{"domain":"facebook.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["Amazonbot","Applebot-Extended","Brightbot","ClaudeBot","Google-Extended","GPTBot","PerplexityBot","PetalBot","Scrapy","uptimerobot","viberbot","YaK","Yandex","Yeti","Applebot","Bingbot","Discordbot","DuckDuckBot","facebookexternalhit","Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News","Google-InspectionTool","LinkedInBot","msnbot","Pinterestbot","Screaming Frog SEO Spider","seznambot","Slurp","teoma","TelegramBot","Twitterbot","*"]}
{"domain":"fbcdn.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["Amazonbot","Applebot-Extended","Brightbot","ClaudeBot","Google-Extended","GPTBot","PerplexityBot","PetalBot","Scrapy","uptimerobot","viberbot","YaK","Yandex","Yeti","Applebot","Bingbot","Discordbot","DuckDuckBot","facebookexternalhit","Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News","Google-InspectionTool","LinkedInBot","msnbot","Pinterestbot","Screaming Frog SEO Spider","seznambot","Slurp","teoma","TelegramBot","Twitterbot","*"]}
{"domain":"twitter.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["Googlebot","Bingbot","facebookexternalhit","Google-Extended","FacebookBot","Discordbot","*"]}
{"domain":"apple.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Baiduspider","HaoSouSpider","Sogou web spider","Sogou inst spider","Sogou spider2"]}
{"domain":"doubleclick.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"microsoft.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"youtube.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google*","*"]}
{"domain":"gstatic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"github.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["bingbot","adidxbot","BingPreview","baidu","*"]}
{"domain":"instagram.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["Amazonbot","Applebot-Extended","Brightbot","ClaudeBot","Google-Extended","GPTBot","PerplexityBot","PetalBot","Scrapy","uptimerobot","viberbot","YaK","Yandex","Yeti","Applebot","Bingbot","Discordbot","DuckDuckBot","facebookexternalhit","Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News","Google-InspectionTool","LinkedInBot","msnbot","Pinterestbot","Screaming Frog SEO Spider","seznambot","Slurp","teoma","TelegramBot","Twitterbot","*"]}
{"domain":"cloudflare.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","Anthropic-AI","Claude-Web","CCBot","PerplexityBot","Cohere-ai"]}
{"domain":"mail.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot","Yandex","Twitterbot"]}
{"domain":"dzen.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","Googlebot","*"]}
{"domain":"ezviz7.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","CERT_HAS_EXPIRED"]}
{"domain":"gtld-servers.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"googleapis.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"wordpress.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","ClaudeBot","anthropic-ai","Google-Extended","Applebot-Extended","PerplexityBot","Bytespider","CCBot","Copilot"]}
{"domain":"akamai.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"akamaiedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"googletagmanager.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"amazonaws.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AdsBot-Google","AdsBot-Google-Mobile","*"]}
{"domain":"googlevideo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"linkedin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User","anthropic-ai"],"open":[],"path_restricted":["Claude-SearchBot"]}],"stanza_token_count":76,"stanza_tokens":["LinkedInBot","Googlebot","Applebot","Bingbot","msnbot","Slurp","Googlebot-Image","baiduspider","Baiduspider","seznambot","Teoma","Yandex","Yeti","msnbot-media","Googlebot-News","Googlebot-Video","Googlebot-Mobile","Mediapartners-Google","AdsBot-Google","DuckDuckBot","daumoa","OrangeBot","OrangeBot-Collector","StackRambler","Twitterbot/1.0","Mail.RU_Bot","naverbot","vebidoobot","MJ12bot","360Spider","Sogou","rogerbot","deepcrawl","Neevabot","coccocbot-web","Zoombot","NewsWhip-LinkedIn-Bot","Cincobot","facebookexternalhit","outbrain","OAI-SearchBot","redditbot","Google-Extended","anthropic-ai","ChatGPT-User","ClaudeBot","Claude-Web","Claude-User","Claude-SearchBot","cohere-ai","GPTBot","Google-CloudVertexBot","PerplexityBot","Perplexity-User","DuckAssistBot","MyCentralAIScraperBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","quillbot.com","BLEXBot","CCBot","DataForSeoBot","news-please","omgili","omgilibot","magpie-crawler","peer39_crawler","Quora-Bot","Scrapy","SeekrBot","SeznamHomepageCrawler","Timpibot","Poseidon Research Crawler","Bytespider","*"]}
{"domain":"domaincontrol.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ENOTFOUND"]}
{"domain":"akadns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"whatsapp.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":17,"stanza_tokens":["Amazonbot","Applebot-Extended","Brightbot","ClaudeBot","Google-Extended","GPTBot","PerplexityBot","PetalBot","Scrapy","uptimerobot","viberbot","YaK","Yandex","Yeti","facebookexternalhit","meta-externalads","*"]}
{"domain":"fastly.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"apple-dns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"trafficmanager.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"microsoftonline.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"aaplimg.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"office.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"appsflyersdk.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"youtu.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"googleusercontent.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"amazon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"x.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["Googlebot","Bingbot","facebookexternalhit","Google-Extended","FacebookBot","Discordbot","*"]}
{"domain":"whatsapp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":17,"stanza_tokens":["Amazonbot","Applebot-Extended","Brightbot","ClaudeBot","Google-Extended","GPTBot","PerplexityBot","PetalBot","Scrapy","uptimerobot","viberbot","YaK","Yandex","Yeti","facebookexternalhit","meta-externalads","*"]}
{"domain":"cloudfront.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hicloudcam.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"wikipedia.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["MJ12bot","Mediapartners-Google*","IsraBot","Orthogaffe","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","fast","wget","grub-client","k2spider","NPBot","WebReaper","SemrushBot","*"]}
{"domain":"chatgpt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["CCBot","img2dataset","Google-Extended","anthropic-ai","Claude-Web","Omgilibot","Omgili","FacebookBot","Bytespider","magpie-crawler","PerplexityBot","Perplexity‑User","*"]}
{"domain":"live.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"icloud.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot","facebookexternalhit"]}
{"domain":"wa.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bing.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["msnbot-media","Twitterbot","*"]}
{"domain":"cloudflare.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Eluta.ca","MS Search 6.0 Robot","gsa-crawler"]}
{"domain":"office.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"vimeo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":25,"stanza_tokens":["*","SeznamBot","msnbot","Bingbot","AdIdxBot","BingPreview","Yandex","GPTBot","Google-Extended","anthropic-ai","Claude-Web","Claudebot","CCBot","CC-Crawler","Common Crawl","CommonCrawl","Diffbot","FacebookBot","Bytespider","Omgilibot","Omgili","ImagesiftBot","Meltwater","Seekr","Grok"]}
{"domain":"netflix.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["*","googlebot","Googlebot-Video","Applebot","bingbot","Baiduspider","Baiduspider-mobile","Baiduspider-video","Baiduspider-image","NaverBot","Y!J-WSC","YJ-WSC","Yeti","Yandex","YandexBot","YandexMobileBot","YandexVideo","YandexWebmaster","YandexSitelinks","SeznamBot","facebookexternalhit","GPTBot","ChatGPT-User","OAI-SearchBot","Google-Extended","ClaudeBot","Claude-User","Claude-SearchBot","PerplexityBot","AdsBot-Google","Twitterbot","Adidxbot","Yahoo Pipes 1.0","Facebot"]}
{"domain":"pinterest.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":315,"stanza_tokens":["2checkout","AASA-Bot","AccessibleWebBot","Adagiobot","adbeat_bot","ADmantX","AdometryBot","Adsense-Snapshot-Google","AdsTxtCrawler","Adyen","AhrefsBot","AhrefsSiteAudit","Akismet","Algolia Crawler","AmazonProductDiscovery","Aolbot-News","APIs-Google","AppInsights","Applebot","AppleNewsBot","Archive-It","archive.org_bot","ArchiveBot","ArchiveTeam-ArchiveBot","AspiegelBot","Audisto Crawler","Awario","Baidu-ADS","Baiduspider","Barkrowler","bingbot","BingPreview","bitlybot","bl.uk_ldfc_bot","BLP_bbot","bordermanager","botify","BrandVerity","Bravebot","BrightEdge Crawler","BrightEdge-Crawler","BUbiNG","CaliperBot","CapitalOneBot","Catchpoint","cf-qualys-scanner","Chrome-Lighthouse","CJBot","ClarityBot","Clickagy Intelligence Bot v2","Cloudflare-AI-Search","CloudflareBrowserRenderingCrawler","coccocbot","ContextualBot","CookieReports","Coveobot","CriteoBot","curly-","cXensebot","Datadog","DataForSeoBot","Daum","demandbase","Discordbot","DotBot","DoubleVerifyBot","Doximity-Pipeline","Dragonbot","Dratabot","DuckAssistBot","DuckDuckBot","DuckDuckGo-Favicons-Bot","DVBot","eContext","ecurly Webhook","EngageBDR","europarchive.org","evidon","ExaleadCloudview","FacebookBot","facebookcatalog","facebookexternalhit","Facebot","FastDAST","FeedBurner","Feedfetcher-Google","FlipboardProxy","FlippBot","Freshping","GeedoProductSearch","Google Favicon","Google News","Google Page Speed Insights","Google Wireless Transcoder","Google-Apps-Script","Google-Certificates-Bridge","Google-CWS","Google-Display-Ads-Bot","Google-Firebase","Google-HotelAdsVerifier"]}
{"domain":"digicert.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":23,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","PerplexityBot","Perplexity-User","ClaudeBot","Claude-SearchBot","Claude-User","Googlebot-Image","AdsBot-Google","Google-Extended","XoviBot","BLEXBot","DotBot","MJ12bot","Xenu's","Xenu's Link Sleuth 1.1c","Zeus","Zeus 32297 Webster Pro V2.9 Win32","Updownerbot","Blekkobot","008"]}
{"domain":"tiktok.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":27,"stanza_tokens":["Baiduspider","360Spider","Sogouspider","Yisouspider","PetalBot","GPTBot","OAI-SearchBot","anthropic-ai","ClaudeBot","PerplexityBot","Google-Extended","Applebot-Extended","meta-externalagent","DuckAssistBot","ChatGPT-User","Gemini-Deep-Research","GoogleAgent-Mariner","Google-NotebookLM","CCBot","Claude-User","Claude-SearchBot","Perplexity-User","AI2Bot","MistralAI-User","Bytespider","*","Bingbot"]}
{"domain":"zoom.us","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["*","Googlebot","LinkedInBot","facebookexternalhit","Twitterbot","PinterestBot","InstagramBot","Slackbot","WhatsApp","redditbot","Tumblr","TikTokBot","Snapchat"]}
{"domain":"azure.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"windowsupdate.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gvt1.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"akam.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tiktokcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gvt2.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"msn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":74,"stanza_tokens":["*","AdsBot-Google","Verity","ias_crawler","Mediapartners-Google","CriteoBot","yandex.com/bot","admantx-ussy04/3.2","TTD-Content","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","Bedrockbot","Bytespider","CCBot","ChatGLM","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","Claude-Web","cohere-ai","cohere-training-data-crawler","Cotoyogi","DeepSeekBot","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Gensparkbot","Google-CloudVertexBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","Grok","iaskspider/2.0","ICC-Crawler","ImagesiftBot","Img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","KunatoCrawler","magpie-crawler","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","MistralAI","MistralAI-User","Momentumbot","OAI-SearchBot","omgili","Omgilibot","Other","PanguBot","Perplexity-ai","PerplexityBot","Perplexity-User","QualifiedBot","Scrapy","Seekr","Sidetrade indexer bot","TikTokSpider","Timpibot","Twitterbot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","WRTNBot","YouBot"]}
{"domain":"adobe.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["gsa-crawler-www","Atomz/1.0","*","Googlebot","AdsBot-Google","AdsBot-Google-Mobile","SearchmetricsBot"]}
{"domain":"myfritz.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","*"]}
{"domain":"googledomains.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"roblox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"edgekey.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"nginx.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yahoo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":46,"stanza_tokens":["*","ADmantX","AlphaBot","anthropic-ai","AwarioRssBot","AwarioSmartBot","BLEXBot","Buzzbot","Bytespider","CCBot","ChatGPT-User","claritybot","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","huggingface","ImagesiftBot","img2dataset","magpie-crawler","Meltwater","Neevabot","news-please","NewsNow","Nutch","omgili","omgilibot","panscient.com","Perplexity-ai","PerplexityBot","PetalBot","PiplBot","scoop.it","Scrapy","Seekr","SentiBot","SeznamBot","TurnitinBot","YouBot","ZumBot","Claude-SearchBot","OAI-SearchBot"]}
{"domain":"goo.gl","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":[400,404]}
{"domain":"spotify.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"skype.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"windows.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"googlesyndication.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"cloud.microsoft","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"cdninstagram.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wordpress.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"discord.gg","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ui.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sentry.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bit.ly","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ntp.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nic.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"yandex.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Bingbot","Yandex","Googlebot"]}
{"domain":"gandi.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":206,"stanza_tokens":["*","facebookexternalhit/1.1","Facebot","AhrefsBot","Alexibot","Aqua_Products","asterias","b2w/0.1","BackDoorBot/1.0","BacklinkCrawler","bdbrandprotect","Black Hole","BlowFish/1.0","boitho.com-dc","Bookmark search tool","BotALot","BotRightHere","BPImageWalker","BuiltBotTough","Bullseye/1.0","BunnySlippers","CazoodleBot","Cegbfeieh","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","ConveraCrawler","Copernic","CopyRightCheck","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","discobot","DittoSpyder","dotbot","Download Ninja","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","ExtractorPro","Ezooms","FairAd Client","Fasterfox","Fetch","findlinks","Flaming AttackBot","Foobot","Gaisbot","GetRight/4.2","Gigabot","grub","grub-client","Harvest/1.5","heritrix","hloader","httplib","HTTrack","HTTrack 3.0","HuaweiSymantecSpider","humanlinks","ichiro","InfoNaviRobot","IRLbot","Iron33/1.0.2","JennyBot","kalooga","Kenjin Spider","Keyword Density/0.9","larbin","LexiBot","LexxeBot","libWeb/clsHTTP","libwww","LinkextractorPro","linko","LinkScan/8.1a Unix","LinkWalker","LNSpiderguy","lwp-trivial","lwp-trivial/1.34","Mata Hari","Microsoft URL Control","Microsoft URL Control - 5.01.4511","Microsoft URL Control - 6.00.8169","Microsoft.URL.Control","MIIxpc","MIIxpc/4.2","Mister PiX","MJ12bot","MLBot","moget","moget/2.1","Mozilla/4.0 (compatible; BullsEye; Windows 95)","MSIECrawler","NaverBot","NerdyBot","Net\\ Vampire","NetAnts"]}
{"domain":"pv-cdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ytimg.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google-analytics.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cloudflare-dns.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"mozilla.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tiktokv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["Yeti"]}
{"domain":"europa.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["bingbot","*"]}
{"domain":"vk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot","Yandex"]}
{"domain":"edgesuite.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"baidu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["Baiduspider","Googlebot","MSNBot","Baiduspider-image","YoudaoBot","Sogou web spider","Sogou inst spider","Sogou spider2","Sogou blog","Sogou News Spider","Sogou Orion spider","ChinasoSpider","Sosospider","yisouspider","EasouSpider","*"]}
{"domain":"qq.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yandex.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"gwfb.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"root-servers.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"reddit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"samsung.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","Googlebot","AdsBot-Google","Googlebot-Image","OAI-SearchBot","GPTBot","ChatGPT-User","PerplexityBot","Google-Extended","Yandex","Sitereport","proximic"]}
{"domain":"apache.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"blogspot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"workers.dev","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"opera.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"office365.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"criteo.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"github.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"b-cdn.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"a2z.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gravatar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":11,"stanza_tokens":["*","GPTBot","CCBot","SentiBot","sentibot","Google-Extended","FacebookBot","omgili","omgilibot","Amazonbot","Bytespider"]}
{"domain":"nginx.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"windows.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"snapchat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":49,"stanza_tokens":["*","Meta-WebIndexer","Meta-ExternalAgent","Meta-ExternalFetcher","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot-Extended","Brightbot 1.0","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Crawlspace","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Googlebot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","Perplexity‑User","PetalBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot"]}
{"domain":"t.me","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"trbcdn.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"vedcdnlb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"registrar-servers.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ENOTFOUND"]}
{"domain":"app-measurement.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"googleadservices.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unity3d.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Baiduspider","Sosospider","sogou spider","*"]}
{"domain":"amazonvideo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":[],"path_restricted":["GPTBot","OAI-SearchBot"]},{"family":"anthropic","blocked":["Claude-User"],"open":[],"path_restricted":["ClaudeBot","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":94,"stanza_tokens":["*","AddSearchBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","Andibot","Anomura","atlassian-bot","Brightbot","BuddyBot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT-User","Claude-User","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-ai","cohere-training-data-crawler","Copilot","CopilotNative","CopilotSapphire","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","EtaoSpider","factset_spyderbot","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Firebase","Google-NotebookLM","GoogleAgent-Mariner","GoogleAgent-Shopping","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","iaskspider","IbouBot","ICC-Crawler","img2dataset","ImagesiftBot","imageSpider","imgproxy","ISSCyberRiskCrawler","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-externalagent","meta-externalfetcher","meta-webindexer","MistralAI-User","netEstate Imprint Crawler","omgili","Panscient","panscient.com","PanguBot","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","Scrapy","SemrushBot-SWA","ShapBot","Sidetrade indexer bot","TavilyBot","TerraCotta","Thinkbot","Timpibot","TwinAgent","VelenPublicWebCrawler","webzio-extended","wpbot","WRTNBot","xAI-Grok","YaK","YouBot","ZanistaBot"]}
{"domain":"cdn77.org","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"amazon-adsystem.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"amazon.dev","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"app-analytics-services.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"msftncsi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"archive.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"prodregistryv2.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"nih.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"forms.gle","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":[400,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"intuit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm","YisouSpider","Baiduspider","Baiduspider+","Baiduspider+(+http://www.baidu.com/search/spider.htm)","Baiduspider/2.0;+http://www.baidu.com/search/spider.html","Baiduspider/2.0","+Baiduspider","+Baiduspider/2.0","+Baiduspider/2.0;++http://www.baidu.com/search/spider.html","Mozilla/5.0(compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)","Baiduspider-image+(+http://www.baidu.com/search/spider.htm)","Mozilla/5.0 (compatible; Sosospider/2.0; +http://help.soso.com/webspider.htm)","Mozilla/5.0 (compatible; JikeSpider; +http://shoulu.jike.com/spider.html)","Twitterbot","*"]}
{"domain":"f5.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dns.google","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"dropbox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Twitterbot","facebookexternalhit","*"]}
{"domain":"okcdn.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"discord.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Twitterbot","nsa"]}
{"domain":"kaspersky.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","GPTBot","ClaudeBot","claude-web","Claude-User","PerplexityBot","Perplexity-User","Google-Extended","Applebot","Applebot-Extended","DuckAssistBot","CCBot","Diffbot","MistralAI-User","YouBot","DeepSeekBot"]}
{"domain":"iiko.it","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"yccdn.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"steamserver.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"shopify.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"static.microsoft","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"godaddy.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"flickr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":47,"stanza_tokens":["Adsbot-Google","Alexabot","AppleBot","archive.org_bot","Assaybot","baiduspider","bitlybot","bingbot","BingPreview","ChatGPT-User","Claude-SearchBot","Claude-User","coccocbot","Discordbot","DuckDuckBot","facebookexternalhit","Google-Site-Verification","Google-Sitemaps","Googlebot","Googlebot-Image","Googlebot-Mobile","Googlebot-News","Googlebot-Video","gsa-crawler","HatenaBlog-bot","ia_archiver","Mediapartners-Google","msnbot","NAVER","OAI-SearchBot","PerplexityBot","PetalBot","Pingdom","Pinterest","redditbot","Slackbot","Slurp","snapchat","Twitterbot","wp.com","Yandex","YandexImages","YandexVideoParser","Yeti","ZoomBot","magpie-crawler","*"]}
{"domain":"applovin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"miit.gov.cn","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"dnsowl.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nytimes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":62,"stanza_tokens":["*","Googlebot","AliyunSecBot","AmazonBot","anthropic-ai","Applebot-Extended","archive.org_bot","AudigentAdBot","AwarioRssBot","AwarioSmartBot","BLEXBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","DuckAssistBot","EchoboxBot","FacebookBot","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","GPTBot","ImagesiftBot","Jetslide","magpie-crawler","Meta-ExternalAgent","meta-externalagent","Meta-ExternalFetcher","meta-externalfetcher","Meta-WebIndexer","meta-webindexer/1.1","MyCentralAIScraperBot","NewsNow","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Perplexity-User","Poseidon Research Crawler","quillbot.com","Quora-Bot","Scrapy","SeekrBot","SeznamHomepageCrawler","TaraGroup Intelligent Bot","Timpibot","TurnitinBot","ViennaTinyBot","YouBot","AmazonAdBot","facebookexternalhit","Twitterbot","RedditBot"]}
{"domain":"spo-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tumblr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":19,"stanza_tokens":["Googlebot","Slurp","Yandex","msnbot","*","CCBot","sentibot","Google-Extended","FacebookBot","omgili","omgilibot","Amazonbot","ClaudeBot","anthropic-ai","ImagesiftBot","Applebot-Extended","TurnitinBot","meta-externalagent","YouBot"]}
{"domain":"azurefd.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"paypal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AdsBot-Google"]}
{"domain":"ax-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dns-parking.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"msftconnecttest.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"adtrafficquality.google","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"mailinabox.email","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"medium.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":9,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","ClaudeBot","FacebookBot","GoogleOther","GPTBot","meta-externalagent"]}
{"domain":"soundcloud.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":15,"stanza_tokens":["anthropic-ai","ClaudeBot","Claude-Web","GPTBot","ChatGPT-User","OAI-SearchBot","CCBot","PerplexityBot","Google-Extended","Applebot-Extended","Bytespider","Amazonbot","Meta-ExternalAgent","cohere-ai","*"]}
{"domain":"reg.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Yandex","Googlebot","CCBot","*"]}
{"domain":"one.one","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vkuserphoto.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":418,"error":null,"attempts":[418,418]}
{"domain":"3gppnetwork.org","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"aws.dev","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ggpht.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nflxso.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"azurewebsites.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNREFUSED"]}
{"domain":"aliyuncs.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"vungle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adobe.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"creativecommons.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"w3.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","W3C-checklink"]}
{"domain":"tm-azurefd.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"userapi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot","Yandex"]}
{"domain":"hichina.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"cdn-apple.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wac-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"example.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"cnn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":77,"stanza_tokens":["AI2Bot","Ai2Bot-Dolma","AliyunSecBot","Amazonbot","amzn-searchbot","amzn-user","anthropic-ai","Applebot-Extended","Archive.org_bot","AwarioRssBot","AwarioSmartBot","Brightbot 1.0","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","cohere-training-data-crawler","Crawlspace","DataForSeoBot","Diffbot","DuckAssistBot","EchoboxBot","Exabot","FacebookBot","FriendlyCrawler","GPTBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","IAB-Tech-Lab","iaskspider/2.0","ICC-Crawler","img2dataset","ISSCyberRiskCrawler","ImagesiftBot","Kangaroo Bot","magpie-crawler","MistralAI-user","MyCentralAIScraperBot","NewsNow","news-please","OAI-SearchBot","omgili","omgilibot","PanguBot","Panscient","PiplBot","PerplexityBot","Perplexity-User","PetalBot","Poseidon Research Crawler","QuillBot","quillbot.com","Quora-Bot","SBIntuitionsBot","Scrapy","SeekrBot","SemrushBot-OCOB","SemrushBot-SWA","SeznamHomepageCrawler","Sidetrade indexer bot","TaraGroup Intelligence Bot","Timpibot","TurnitinBot","VelenPublicWebCrawler","ViennaTinyBot","Webzio-Extended","YandexAdditional","YandexAdditionalBot","YouBot","*","Googlebot-News"]}
{"domain":"epicgames.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bytefcdn-oversea.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sfx.ms","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"forbes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot"],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":25,"stanza_tokens":["*","googlebot-news","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ClaudeBot","cohere-ai","Diffbot","FacebookBot","FriendlyCrawler","GPTBot","ImagesiftBot","meta-externalagent","Meta-ExternalAgent","OAI-SearchBot","omgili","omgilibot","PerplexityBot","AhrefsBot","AhrefsSiteAudit","Claude-User","Claude-SearchBot","AmazonAdBot"]}
{"domain":"rubiconproject.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"weather.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["ChatGPT-User"],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":34,"stanza_tokens":["ChatGPT-User","AmazonAdBot","*","CriteoBot/0.1","Amazonbot","anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ClaudeBot","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","ImagesiftBot","magpie-crawler","Meta-ExternalAgent","meta-externalagent","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Quora-Bot","Scrapy","TurnitinBot"]}
{"domain":"pki.goog","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"theguardian.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":39,"stanza_tokens":["*","Mediapartners-Google","NewsNow","CCBot","TurnitinBot","PetalBot","MoodleBot","FacebookBot","Bytespider","Mojeek","JenkersBot","Seekr","YouBot","Arquivo-web-crawler","coccocbot-web","SeznamBot","PerplexityBot","yacy","anthropic-ai","ClaudeBot","Claude-SearchBot","Claude-User","AwarioRssBot","AwarioSmartBot","SentiOne","ImageSift","Applebot-Extended","YandexAdditional","YandexAdditionalBot","scalepostAI","Buck","meta-externalagent","Amazonbot","amazon-QBusiness","DuckAssistBot","Google-CloudVertexBot","Amzn-SearchBot","AhrefsBot","AhrefsSiteAudit"]}
{"domain":"taboola.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"webex.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bbc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":40,"stanza_tokens":["*","Amazonbot","magpie-crawler","CCBot","omgili","omgilibot","Claude-Web","ClaudeBot","anthropic-ai","cohere-ai","Bytespider","PetalBot","Scrapy","Applebot-Extended","GPTBot","ChatGPT-User","Google-Extended","PerplexityBot","Perplexity-User","Google-CloudVertexBot","meta-externalagent","OAI-SearchBot","YandexAdditional","YandexAdditionalBot","TurnitinBot","Brightbot","ApifyBot","ApifyWebsiteContentCrawler","Diffbot","Diffbot-User","ExaBot","TavilyBot","ShapBot","YouBot","FirecrawlAgent","Amzn-SearchBot","Amzn-User","ProRataInc","CloudflareBrowserRenderingCrawler","AhrefsBot"]}
{"domain":"edgcdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"comcast.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"doi.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"pubmatic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xiaomi.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"t.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["twitterbot","Googlebot","Bingbot","*"]}
{"domain":"oracle.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"avast.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","PerplexityBot","OAI-SearchBot","ChatGPT-User","GPTBot","Perplexity-User","Claude-User","ClaudeBot"]}
{"domain":"ebay.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":20,"stanza_tokens":["*","bingbot","AdsBot-Google","AdIdxBot","Twitterbot","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","PerplexityBot","anthropic-ai","AmazonBot","meta-externalagent","GPTBot","Applebot-Extended","OAI-SearchBot","ChatGPT-User","Claude-SearchBot","Claude-User","OAI-AdsBot"]}
{"domain":"gmail.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sourceforge.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","AmazonAdBot","*"]}
{"domain":"openai.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"t-online.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","PerplexityBot","ClaudeBot","Applebot-Extended","Bytespider","CCBot","Google-Extended","Meta-ExternalAgent","meta-externalagent"]}
{"domain":"cpanel.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ttdns2.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"youtube-nocookie.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google*","*"]}
{"domain":"twitch.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Amazonbot","Amzn-SearchBot","Amzn-User","*"]}
{"domain":"sciencedirect.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"nominetdns.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bbc.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":40,"stanza_tokens":["*","Amazonbot","magpie-crawler","CCBot","omgili","omgilibot","ClaudeBot","Claude-Web","anthropic-ai","cohere-ai","Bytespider","PetalBot","Scrapy","Applebot-Extended","GPTBot","ChatGPT-User","Google-CloudVertexBot","Google-Extended","PerplexityBot","Perplexity-User","meta-externalagent","OAI-SearchBot","YandexAdditional","YandexAdditionalBot","TurnitinBot","Brightbot","ApifyBot","ApifyWebsiteContentCrawler","Diffbot","Diffbot-User","ExaBot","TavilyBot","ShapBot","YouBot","FirecrawlAgent","Amzn-SearchBot","Amzn-User","ProRataInc","CloudflareBrowserRenderingCrawler","AhrefsBot"]}
{"domain":"azureedge.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"researchgate.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"outlook.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"nist.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"stripe.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["ia_archiver","*","rogerbot"]}
{"domain":"fastly-edge.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"oxylabs.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mit.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"akamaized.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"spov-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"doubleverify.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adsafeprotected.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"android.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","GPTBot","OAI-SearchBot","Google-Extended","PerplexityBot","ClaudeBot","Claude-SearchBot","anthropic-ai","cohere-ai","CCBot","Diffbot","Applebot-Extended","Applebot","YouBot","MistralAI-User"]}
{"domain":"playstation.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"shopifysvc.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mts.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"canva.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":32,"stanza_tokens":["*","AdsBot-Google","AdsIdxBot","Google-InspectionTool","facebookexternalhit","LinkedInBot","PerplexityBot","Perplexity-User","Canva-Slackapp-LinkExpanding","Canva-Teamsapp-LinkExpanding","GPTBot","ClaudeBot","CCBot","Applebot-Extended","anthropic-ai","Claude-Web","Bytespider","meta-externalagent","ImagesiftBot","Timpibot","Scrapy","Ai2Bot","Diffbot","img2dataset","ICC-Crawler","FacebookBot","VelenPublicWebCrawler","FriendlyCrawler","omgili","Webzio-Extended","Ai2Bot-Dolma","Twitterbot"]}
{"domain":"cisco.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["cisco-googlebot-enterprise","GPTBot","CCBot","Google-Extended","ClaudeBot","Googlebot","Googlebot Smartphone","Google AdsBot","Bingbot","*","Yandex"]}
{"domain":"adsrvr.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[403,200]}
{"domain":"linktr.ee","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":59,"stanza_tokens":["Applebot","Applebot-Extended","baiduspider","Bingbot","Discordbot","facebookexternalhit","Googlebot","Googlebot-Image","Google-InspectionTool","ia_archiver","LinkedInBot","msnbot","Naverbot","Pinterestbot","Screaming Frog SEO Spider","seznambot","Slurp","teoma","TelegramBot","Twitterbot","Yandex","Yeti","SnapchatAdsBot","SemrushBot","OAI-SearchBot","ChatGPT-User","ChatGPT-User/2.0","ChatGPT Agent","OpenAI","GPTBot","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","PerplexityBot","Perplexity-User","MistralAI-User","MistralAI-User/1.0","Meta-ExternalFetcher","Google-Extended","Google","Google-NotebookLM","NotebookLM","Gemini-Deep-Research","GoogleAgent-Mariner","Google-CloudVertexBot","Amazonbot","amazon-kendra","AmazonBuyForMe","NovaAct","CCBot","atlassian-bot","Cloudflare-AutoRAG","KlaviyoAIBot","ShapBot","Bravebot","DuckAssistBot","Devin","*"]}
{"domain":"pages.dev","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"booking.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"ubuntu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","Google-Extended","meta-externalagent","PerplexityBot","cohere-ai","Bytespider"]}
{"domain":"who.int","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":527,"stanza_tokens":["360Spider","404checker","404enemy","80legs","Abonti","Aboundex","Acunetix","ADmantX","AfD-Verbotsverfahren","AhrefsBot","AIBOT","AiHitBot","Aipbot","Alexibot","Alligator","AllSubmitter","AlphaBot","Anarchie","Apexoo","ASPSeek","Asterias","Attach","autoemailspider","BackDoorBot","Backlink-Ceck","backlink-check","BacklinkCrawler","BackStreet","BackWeb","Badass","Bandit","Barkrowler","BatchFTP","Battleztar Bazinga","BBBike","BDCbot","BDFetch","BetaBot","Bigfoot","Bitacle","Blackboard","Black Hole","BlackWidow","BLEXBot","Blow","BlowFish","Boardreader","Bolt","BotALot","Brandprotect","Brandwatch","Bubing","Buddy","BuiltBotTough","BuiltWith","Bullseye","BunnySlippers","BuzzSumo","Calculon","CATExplorador","CazoodleBot","CCBot","Cegbfeieh","CheeseBot","CherryPicker","ChinaClaw","Chlooe","Claritybot","Cliqzbot","Cloud mapping","coccocbot-web","Cogentbot","cognitiveseo","Collector","com.plumanalytics","Copier","CopyRightCheck","Copyscape","Cosmos","Craftbot","crawler4j","crawler.feedback","CrazyWebCrawler","Crescent","CSHttp","Curious","Custo","DatabaseDriverMysqli","DataCha0s","DBLBot","demandbase-bot","Demon","Deusu","Devil","Digincore","DigitalPebble","DIIbot","Dirbuster","Disco","Discobot"]}
{"domain":"imdb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":36,"stanza_tokens":["Googlebot","Googlebot-Image","Googlebot-Video","Google-PageRenderer","Google-InspectionTool","Google-Safety","Google-Site-Verification","AdsBot-Google","AdsBot-Google-Mobile","Chrome-Lighthouse","Storebot-Google","Mediapartners-Google","GoogleStructuredData","Google-Read-Aloud","bingbot","Applebot","DuckDuckBot","Baiduspider","Sogou","MojeekBot","facebookexternalhit","Twitterbot","Pinterestbot","LinkedInBot","WhatsApp","TelegramBot","Discordbot","Snapchat","Slackbot-LinkExpanding","AmazonAdBot","Amazonbot","Amzn-SearchBot","Amzn-User","ClarityBot","Citoid","*"]}
{"domain":"forter.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":25,"stanza_tokens":["GPTBot","ClaudeBot","anthropic-ai","Google-Extended","Applebot-Extended","ChatGPT-User","OAI-SearchBot","Claude-User","Claude-SearchBot","PerplexityBot","Perplexity-User","DeepSeekBot","Amazonbot","cohere-ai","MistralAI-User","OpenClaw","CCBot","Bytespider","FacebookBot","meta-externalagent","ImagesiftBot","Diffbot","omgili","omgilibot","*"]}
{"domain":"roku.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"afternic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hubspot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"scorecardresearch.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"wsdvs.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"miui.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wixsite.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AdsBot-Google-Mobile","AdsBot-Google","PetalBot","dotbot","AhrefsBot"]}
{"domain":"virginm.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ksyuncdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"facebook.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wikimedia.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"meraki.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":72,"stanza_tokens":["360Spider","360Spider-Image","360Spider-Video","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","adidxbot","Applebot","AppleNewsBot","Baiduspider","Baiduspider-image","Baiduspider-news","Baiduspider-video","bingbot","BingPreview","BublupBot","CCBot","Cliqzbot","coccoc","coccocbot-image","coccocbot-web","Daumoa","Dazoobot","DeuSu","DuckDuckBot","DuckDuckGo-Favicons-Bot","EuripBot","Exploratodo","Facebot","Feedly","Findxbot","Googlebot","Googlebot-Image","Googlebot-Mobile","Googlebot-News","Googlebot-Video","HaoSouSpider","ichiro","istellabot","JikeSpider","Lycos","Mail.Ru","Mediapartners-Google","MojeekBot","msnbot","msnbot-media","OrangeBot","Pinterest","Plukkie","Qwantify","Rambler","SeznamBot","Sosospider","Slurp","Sogou blog","Sogou inst spider","Sogou News Spider","Sogou Orion spider","Sogou spider2","Sogou web spider","SputnikBot","Teoma","Twitterbot","wotbox","yacybot","Yandex","YandexMobileBot","Yeti","YioopBot","yoozBot","YoudaoBot","*"]}
{"domain":"vecdnlb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"alibabadns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"libp2p.direct","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dropcatch.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"telegram.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"tinyurl.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ibm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","Fast corporate crawler","008","magpie-crawler","magpie-crawler/1.1","MagpieRSS","psnsearch","HTTrack","Acrolinx-bot","amazon-Quick-on-behalf-of"]}
{"domain":"www.gov.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","meta-externalagent","AhrefsBot","deepcrawl","MS Search 6.0 Robot"]}
{"domain":"myshopify.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"hcaptcha.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"springer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"omtrdc.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"capcutapi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ttvnw.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"deere.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tiktokcdn-us.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","AdsBot-Google","Googlebot-Image","AdIdxBot","MSNBot-Media","GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Google-Extended","PerplexityBot","Perplexity-User"]}
{"domain":"myhuaweicloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","CERT_HAS_EXPIRED"]}
{"domain":"lsrelayaccess.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ECONNRESET"]}
{"domain":"salesforce.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["MS Search 4.0 Robot","Mozilla/4.0 (compatible; MSIE 4.01; Windows NT; MS Search 4.0 Robot)","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)","Mozilla/4.0 (compatible; MSIE 4.01; Windows NT; MS Search 5.0 Robot)","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 5.0 Robot)","*","AdsBot-Google","Acunetix Web Vulnerability Scanner","Acunetix Security Scanner"]}
{"domain":"mozilla.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"namebrightdns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dnsmadeeasy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"reuters.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":87,"stanza_tokens":["AASA-Bot","ADmantX","AdsBot-Google","AdsBot-Google-Mobile","AmazonAdBot","Amzn-User","Applebot","AppleNewsBot","Bingbot","BingPreview","BrightEdge","BrightEdgeOnCrawl","CensysInspect","ChatGPT-User","Cision","Clickagy","Concert","ContextualBot","CriteoBot","DatadogSynthetics","datadome-pageprotect-scanner","Discordbot","doubleverify","DuckDuckBot","ElevenlabsBot","Embedly","facebookexternalhit","Googlebot","Googlebot Smartphone","Googlebot-News","Google-Display-Ads-Bot","Google-InspectionTool","GoogleOther","Google-Read-Aloud","Google-Safety","Google-Site-Verification","GTmetrix","GumGumBot","ias_crawler","Iframely","leiki","LinkedInBot","LinkTiger","Mantisbot","Mediapartners-Google","meta-externalads","meta-webindexer","MicrosoftPreview","MJ12bot","Moreover","msnbot","NFBNewslineRobot","OAI-SearchBot","Oncrawl","Opebot-v","Opoint","Optimizer","outbrain","Pinterestbot","Prerender","Proofpoint","proximic","PubMatic Crawler Bot","Quantcastbot","Qwantbot","Reuters SEO Screaming Frog Spider 007","Reuters-NAUWI","Scom-Crawler-For-Reuters","SinceraSyntheticUser","Slurp","SmartologyBot","snews","SocialFlow","StatusCake","Storebot-Google","Stripebot","TTD-Content","Twitterbot","URLDefense","Verity","vuln_scan_by_trustedsite_com_halo_security","WISEbot","Xenu Link Sleuth","Yahoo Link Preview","Yahoo! JAPAN","YahooMailProxy","*"]}
{"domain":"alibaba.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":6,"stanza_tokens":["*","bingbot","Googlebot","Googlebot-image","GPTBot","Google-Extended"]}
{"domain":"alidns.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"casalemedia.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"inmobi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lencr.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ampproject.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"telekom.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"byteoversea.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"media-amazon.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"harvard.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"steampowered.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mzstatic.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"launchdarkly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"samsungcloud.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nature.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":23,"stanza_tokens":["*","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ClaudeBot","FacebookBot","GPTBot","Google-Extended","GoogleOther","PerplexityBot","YouBot","meta-externalagent","ImagesiftBot","cohere-ai","Diffbot","FriendlyCrawler","img2dataset","omgili","omgilibot","Claude-Web","ChatGPT-User"]}
{"domain":"etsy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Spinn3r","AdsBot-Google-Mobile","*"]}
{"domain":"googleblog.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"telecid.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",403]}
{"domain":"mtgglobals.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"ok.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"samsungqbe.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"macromedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["gsa-crawler-www","Atomz/1.0","*","Googlebot","AdsBot-Google","AdsBot-Google-Mobile","SearchmetricsBot"]}
{"domain":"name-services.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"arubanetworks.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["CERT_HAS_EXPIRED",403]}
{"domain":"appsflyer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"weebly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["NerdyBot","*"]}
{"domain":"clarity.ms","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",404]}
{"domain":"media.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"php.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"heytapmobile.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nr-data.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"issuu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":47,"stanza_tokens":["AddSearchBot","AwarioBot","AwarioRssBot","AwarioSmartBot","BLEXBot","BomboraBot","CheckLinks","Clickagy Intelligence Bot","cocolyzebot","DataForSeoBot","DomainStatsBot","DotBot","FindLinks","getlinkinfo.com","harsilbot","HubSpot Crawler","linkCheck","LinkWalker","MBCrawler","MJ12bot","Morningscore Bot","nooze-bot","OpenindexSpider","OpenWebSpider","PetalBot","Qwantify","Qwantify-prod","RankFlex","Rogerbot","Screaming Frog SEO Spider","SearchmetricsBot","SeekportBot","SeekrBot","SemrushBot-BA","SemrushBot-OCOB","SEO-Checker-bot","SeobilityBot","SeoSiteCheckup","SEOdiver Bot","SEOkicks-Robot","SEOstats","SiteAuditBot","trendictionbot","webgains-bot","www.deadlinkchecker.com","YaK","*"]}
{"domain":"weibo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":17,"stanza_tokens":["Baiduspider","360Spider","Googlebot","Sogou web spider","bingbot","smspider","HaosouSpider","YisouSpider","Bytespider","DeepSeek","OAI-SearchBot","ChatGPT-User","Kimi","Doubao","meta-externalagent","ChatGLM","*"]}
{"domain":"cdc.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Roverbot","EmailSiphon","MindSpider"]}
{"domain":"quickconnect.to","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"wiley.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vk.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"amazon.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"checkpoint.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ea.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","AI2Bot","Bytespider","FriendlyCrawler","ICC-Crawler","cohere-ai","cohere-training-data-crawler","Meta-ExternalAgent"]}
{"domain":"duckdns.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"autodesk.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"zendesk.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"adnxs.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"EHOSTUNREACH"]}
{"domain":"shalltry.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,404]}
{"domain":"slack.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"drom.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AhrefsBot","SemrushBot","MegaIndex","Googlebot","Yandex","*"]}
{"domain":"bilibili.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wildberries.ru","fetch_ok":false,"usable":false,"reason":"other","status":498,"error":null,"attempts":[498,498]}
{"domain":"crashlytics.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"akamaihd.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"smartadserver.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"jomodns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"opendns.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"launchpad.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":[],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":51,"stanza_tokens":["GPTBot","ClaudeBot","Claude-User","Claude-SearchBot","CCBot","Google-Extended","Applebot-Extended","Facebookbot","Meta-ExternalAgent","Meta-ExternalFetcher","diffbot","PerplexityBot","Perplexity‑User","Omgili","Omgilibot","webzio-extended","ImagesiftBot","Bytespider","TikTokSpider","Amazonbot","Youbot","SemrushBot-OCOB","Petalbot","VelenPublicWebCrawler","TurnitinBot","Timpibot","OAI-SearchBot","ICC-Crawler","AI2Bot","AI2Bot-Dolma","DataForSeoBot","AwarioBot","AwarioSmartBot","AwarioRssBot","Google-CloudVertexBot","PanguBot","Kangaroo Bot","Sentibot","img2dataset","Meltwater","Seekr","peer39_crawler","cohere-ai","cohere-training-data-crawler","DuckAssistBot","Scrapy","Cotoyogi","aiHitBot","Factset_spyderbot","FirecrawlAgent","*"]}
{"domain":"gnu.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":27,"stanza_tokens":["*","MJ12bot","DataForSeoBot","BLEXBot","AhrefsBot","barkrowler","Screaming Frog SEO Spider","ZoomBot","magpie-crawler","DotBot","rogerbot","SemrushBot","SiteAuditBot","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SplitSignalBot","SemrushBot-OCOB","JamesBOT","Oncrawl","AwarioRssBot","AwarioSmartBot","AwarioBot","serpstatbot","netEstate NE Crawler","panscient.com","AcademicBotRTU"]}
{"domain":"wsj.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":58,"stanza_tokens":["*","googlebot","googlebot-image","GoogleOther","Googlebot-Video","Google-InspectionTool","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","Storebot-Google","google-read-aloud","mediapartners-google","bingbot","msnbot","bingpreview","slurp","yahoo","baiduspider","Pinterestbot","Yeti","MojeekBot","360Spider","google-cloudvertexbot","duckduckbot","Applebot","flipboard","qwantbot","SeznamBot","proximic","admantx","thetradedesk","outbrain","ias_crawler","AmazonAdBot","pubmatic","smartologybot","parselybot","Screaming Frog SEO Spider","AhrefsBot","SemrushBot","SimilarWebBot","SISTRIX","botify","Chrome-Lighthouse","ChatGPT-User","GPTBot","OAI-SearchBot","facebookexternalhit","facebot","twitterbot","linkedinbot","snapchat","sentry","Iframely","Vocabtracker","EpvzCrawl6194680250","Citoid","ZoteroTranslationServer"]}
{"domain":"trustpilot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":44,"stanza_tokens":["Googlebot","Mediapartners-Google","Googlebot-Mobile","AdsBot-Google","AdsBot-Google-Mobile","bingbot","msnbot","ia_archiver","Alexabot","ScoutJet","Slurp","Twitterbot","Baiduspider","FacebookBot","facebookexternalhit","DuckDuckBot","ChatGPT-User","OAI-SearchBot","PerplexityBot","Claude-User","Claude-SearchBot","GPTBot","Google-Extended","ClaudeBot","sentibot","Yandex","AwarioRssBot","AwarioSmartBot","ImagesiftBot","CCBot","Applebot-Extended","anthropic-ai","Omgilibot","Omgili","Diffbot","Bytespider","Kangaroo Bot","cohere-ai","Meta-ExternalAgent","Meta-ExternalFetcher","Timpibot","Webzio-Extended","YouBot","*"]}
{"domain":"sharepoint.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"ripn.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ETIMEDOUT","TimeoutError"]}
{"domain":"aliyun.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"selectel.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","*"]}
{"domain":"cdnbuild.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"enacdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"un.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"nasa.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["usasearch","*"]}
{"domain":"allawnos.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"live-video.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"onelink.me","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"washingtonpost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":35,"stanza_tokens":["*","Twitterbot","SemrushBot","Amazonbot","AhrefsBot","anthropic-ai","Applebot-Extended","archive.org_bot","ArchiveBot","Arquivo-web-crawler","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","Claude-Web","ClaudeBot","cohere-training-data-crawler","DataForSeoBot","Diffbot","europarchive.org","FacebookBot","heritrix","ia_archiver","ia_archiver-web.archive.org","ImagesiftBot","LinkArchiver","meta-externalagent","meta-externalfetcher","Nicecrawler","omgili","omgilibot","PerplexityBot","Primalbot","web-archive-net.com.bot","YandexBot"]}
{"domain":"dell.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Baiduspider","Sogou web spider","Sogou inst spider","Sogou spider2","HaoSouSpider","Bytespider","AmazonAdBot","360Spider"]}
{"domain":"stanford.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rzone.de","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bloomberg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":97,"stanza_tokens":["*","Google-Extended","Mediapartners-Google","Spinn3r","Googlebot-News","Twitterbot","GPTBot","CCBot","Python-urllib","python-requests","Python-http-client","ChatGPT-User","Claude-Web","anthropic-ai","PerplexityBot","YouBot","ClaudeBot","Claude-User","Claude-SearchBot","OAI-SearchBot","DuckAssistBot","cohere-ai","Google-Apps","Google-Apps-Script","AppEngine-Google","Google-Cloud","Google-CloudVertexBot","AmazonBot","AmazonAdBot","ByteSpider","Meta-ExternalAgent","Meta-ExternalFetcher","Meta-ExternalAgent-Image","Applebot-Extended","Bytespider-Image","Feedly","FeedlyBot","FeedlyApp","MWFeedParser","comscore","Comscore","HubSpot","hubspot","Criteo","criteo-bot","Peer39_Crawler","rogerbot","linkdexbot","NTENT","AndersPinkBot","IndeedBot","MJ12bot","PetalBot","YisouSpider","360Spider","Qwantify","ToutiaoSpider","Screaming Frog SEO Spider","PRTG","FreshpingBot","Panopta","DatadogSynthetics","Rackspace","censys","burp","Burp","check_http","DotcomMonitor","WatchSumo","WormlyBot","CalibreBot","AudistoBot","Hatena","Hatena-Bookmark","EveryoneSocialBot","Diffbot","Netvibes","WebCEO","Postano","RebelMouse","Muck-Rack","InstapaperViewer","Twurly","LivelapBot","DatagnionBot","Linespider","Discourse","Medusa","pingback","WordPress","wp_ping","MauiBot","ltx71","WeSEE","halebot","BrightBot","UnityPlayer"]}
{"domain":"rocket-cdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNREFUSED"]}
{"domain":"naver.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xboxlive.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"google.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ozon.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TypeError","attempts":["TypeError","TypeError"]}
{"domain":"ezvizlife.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"calendly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","ia_archiver","CCBot"]}
{"domain":"tradingview.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":17,"stanza_tokens":["*","bingbot","Yandex","Google-Extended","Google-CloudVertexBot","GoogleOther","Applebot-Extended","Amazonbot","meta-externalagent","ClaudeBot","PerplexityBot","cohere-training-data-crawler","OmgiliBot","AI2Bot","Bytespider","TikTokspider","DeepSeekBot"]}
{"domain":"telekom.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"demdex.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unsplash.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot"],"open":[],"path_restricted":["Claude-User","anthropic-ai"]}],"stanza_token_count":22,"stanza_tokens":["*","ImagesiftBot","Claude-SearchBot","ClaudeBot","Bytespider","AwarioRssBot","AwarioSmartBot","dotbot","AhrefsBot","AhrefsSiteAudit","MJ12bot","SemrushBot-BA","SemrushBot-ESI","SemrushBot-FT","SemrushBot-OCOB","SemrushBot-SI","SemrushBot-SWA","SiteAuditBot","BrightEdge","proximic","DataForSeoBot","SeobilityBot"]}
{"domain":"trueconf.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"samsungcloudsolution.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"plesk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nvidia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":60,"stanza_tokens":["ai2bot","anthropic-ai","Amazonbot","Amazon-kendra","Applebot-extended","AutoGPT","BedrockBot","Bytespider","CCBot","Chatglm-spider","ChatGPT","ChatGPT-User","Claudebot","Claude-user","Claude-search","Claude-web","Cohere-ai","Cotoyogi","Deepseek","Deepseek Chat","Deepseekbot","DuckAssistBot","DuckDuckBot","FacebookBot","Geminiios","Google-Extended","Googleagent-manier","GPTBot","Grok","ia-archiver","iaskbot","iaskspider","ICC-crawler","Linerbot","Metaaib","Meta-externalagent","Meta-externalfetcher","Meta-llama","Mistralai-user","NovaAct","OAI-Searchbot","Pangubot","Perplexity","PerplexityBot","Perplexity-user","Phindbot","PoeBot","Qwen","Sogou","Timpibot","Tongyi","Wenxiaobai","Xinghuo","Youbot","Yuanbaobot","Googlebot","Googlebot Smartphone","Google AdsBot","Bingbot","*"]}
{"domain":"gitlab.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"douyincdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"yandex.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","yandex","Twitterbot","Applebot"]}
{"domain":"dailymotion.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","Googlebot","Mediapartners-Google","YandexBot","YandexVideo","msnbot-media","Botify"]}
{"domain":"vivoglobal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kwai-pro.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"line.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Twitterbot","Mediapartners-Google","*"]}
{"domain":"kaspersky-labs.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","UNABLE_TO_GET_ISSUER_CERT_LOCALLY"]}
{"domain":"awswaf.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ip-api.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cookiedatabase.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"CERT_HAS_EXPIRED"]}
{"domain":"tiktokv.us","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ibyteimg.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"heytapdl.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tp-link.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"indeed.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":66,"stanza_tokens":["*","Googlebot","Google-Extended","ChatGPT-User","Bingbot","Slurp","DuckDuckBot","YahooJP","PetalBot","PerplexityBot","DuckAssistBot","Perplexity-User","YouBot","Claude-User","Gemini-Deep-Research","Claude-SearchBot","OAI-SearchBot","Google-Agent","Google-CloudVertexBot","Meta-WebIndexer","Meta-ExternalFetcher","xAI-Grok","Grok-DeepSearch","Mediapartners-Google","AdsBot-Google","AdsBot-Google-Mobile","adidxbot","Googlebot-Image","msnbot-media","OmniExplorer_Bot","ia_archiver","008","JobdiggerSpider","Cliqzbot","GoogleOther","omgili","omgilibot","Webzio","Webzio-Extended","Scrapy","MyCentralAIScraperBot","facebookexternalhit","WhatsApp","snapchat","TelegramBot","Twitterbot","Slackbot-LinkExpanding","GPTBot","CCBot","anthropic-ai","FacebookBot","AmazonBot","Applebot-Extended","Bytespider","Baiduspider","cohere-training-data-crawler","FriendlyCrawler","img2dataset","Meta-ExternalAgent","ClaudeBot","DeepSeekBot","GrokBot","Diffbot","AI2Bot","PanguBot","Timpibot"]}
{"domain":"amazon.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"temu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":33,"stanza_tokens":["Googlebot","Googlebot-Image","Bingbot","Google-Extended","GPTBot","OAI-SearchBot","ChatGPT-User","PerplexityBot","ClaudeBot","Claude-User","Claude-SearchBot","AdsBot-Google","AdIdxBot","Applebot","Pinterestbot","Twitterbot","FacebookBot","facebookexternalhit","*","Slurp","Y!J-BRU/VSIDX","Y!J-BRW","Y!J-ASR","Y!J-MMP/dscv","Y!J-WSC","Y!J-DLC","YJ-BRU/VSIDX","YJ-BRW","YJ-ASR","YJ-MMP/dscv","YJ-WSC","YJ-DLC","Bytespider"]}
{"domain":"yahoo.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":3,"stanza_tokens":["Google-Extended","GPTBot","*"]}
{"domain":"adobe.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"globo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":4,"stanza_tokens":["*","CCBot","GPTBot","Google-Extended"]}
{"domain":"ngenix.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"netangels.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"ivi.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","*","Twitterbot"]}
{"domain":"rbxcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"datadoghq.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aliyuncsslbintl.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"debian.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"tiktokcdn-eu.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"aiv-delivery.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"xcal.tv","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tiktokv.eu","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pixabay.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":19,"stanza_tokens":["*","008","AhrefsBot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","dotbot","Google-Extended","GPTBot","linguatools","MJ12bot","Ocelli","Riddlerbot","ShopWiki","sistrix","sitebot","WBSearchBot","www.integromedb.org/Crawler"]}
{"domain":"steamcommunity.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bsky.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"supercell.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"huawei.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"crpt.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"businessinsider.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":13,"stanza_tokens":["*","AmazonAdBot","anthropic-ai","AwarioSmartBot","Bytespider","CCBot","ClaudeBot","DataForSeoBot","Diffbot","newspaper","peer39_crawler","Scrapy","Timpibot"]}
{"domain":"avcdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"espn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":11,"stanza_tokens":["claritybot","GPTBot","Google-Extended","CCBot","ChatGPT-User","anthropic-ai","Omgilibot","Omgili","FacebookBot","Bytespider","*"]}
{"domain":"duckduckgo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"brave.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pangle.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"online-metrix.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"163.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"beian.gov.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNREFUSED"]}
{"domain":"intel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":25,"stanza_tokens":["OmtrBot/1.0","*","SEOkicks","coccocbot","AhrefsBot","SemrushBot","trendictionbot","serpstatbot","startmebot","HatenaBlog-bot","Cincraw","InfoTigerBot","snapchat","Superfeedr","YaK","digitalshadowsbot","Magus","Quora-Bot","INETDEX-BOT","DataForSeoBot","SerendeputyBot","Deskyobot","MJ12bot","BrightEdge","ICC-Crawler"]}
{"domain":"statista.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":72,"stanza_tokens":["*","YisouSpider","Sogou web spider/4.0","Sogou web spider","sogou spider","AhrefsBot","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","Brightbot 1.0","Amazonbot","anthropic-ai","Claude-Web","ClaudeBot","Operator","GPTBot","cohere-training-data-crawler","cohere-ai","Meta-ExternalFetcher","Meta-ExternalAgent","FacebookBot","Google-Extended","GoogleOther","GoogleOther-Video","GoogleOther-Image","Google-CloudVertexBot","PerplexityBot","Perplexity-User","AI2Bot","Ai2Bot-Dolma","Applebot-Extended","Aranet-SearchBot","archive.org_bot","Arquivo-web-crawler","Bytespider","CCBot","Cincraw","Diffbot","DuckAssistBot","FriendlyCrawler","ia_archiver","ia_archiver-web.archive.org","ImagesiftBot","img2dataset","Kangaroo LLM","LinerBot","Magpie-crawler","omgili","omgilibot","PanguBot","Peer39_crawler","TerraCotta","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot"]}
{"domain":"kwaipros.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"cnbc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","anthropic-ai"],"open":[],"path_restricted":["Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":52,"stanza_tokens":["googlebot","GPTBot","PerplexityBot","Amazonbot","anthropic-ai","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","HuggingFaceBot","ChatGPT-User","ClaudeBot","Claude-Web","DataForSeoBot","Diffbot","FacebookBot","Google-Extended","Google-NotebookLM","Google-Agent","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","Scrapy","TurnitinBot","cohere-ai","Applebot-Extended","Meta-ExternalAgent","meta-externalagent","OAI-SearchBot","Claude-User","polymarket-news-bot","KalshiBot","polymarket-bot","OpenClaw","gold-ai-event-bridge","TradingBot","trading-pipeline","kalshi-bot","BivariateKalshiPipeline","Falcon Trading Platform","StockMarketAnalyser","FinanceApp","polyclaw-ingester","IsoxyTrader","Neo Trading Bot","MerlinChartWizard","investment-dashboard","*"]}
{"domain":"wp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"go.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":3,"stanza_tokens":["*","GPTBot","Google-Extended"]}
{"domain":"sina.com.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"edgecdn.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"taobao.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"telegram.me","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"vkontakte.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"amazontrust.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"dual-s-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"shifen.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"awsglobalaccelerator.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cdn20.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sc-gw.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"eventbrite.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":139,"stanza_tokens":["*","Facebot","Slurp","CCBot","AhrefsBot","Balihoo","BotRightHere","WebZip","larbin","b2w/0.1","Copernic","psbot","Python-urllib","NetMechanic","URL_Spider_Pro","CherryPicker","EmailCollector","EmailSiphon","WebBandit","EmailWolf","ExtractorPro","CopyRightCheck","Crescent","SiteSnagger","ProWebWalker","CheeseBot","LNSpiderguy","Alexibot","Teleport","TeleportPro","MIIxpc","Telesoft","Website Quester","WebZip/4.0","WebStripper","WebSauger","WebCopier","NetAnts","Mister PiX","WebAuto","TheNomad","WWW-Collector-E","RMA","libWeb/clsHTTP","asterias","httplib","turingos","spanner","InfoNaviRobot","Harvest/1.5","Bullseye/1.0","Mozilla/4.0 (compatible; BullsEye; Windows 95)","Crescent Internet ToolPak HTTP OLE Control v.1.0","CherryPickerSE/1.0","CherryPickerElite/1.0","WebBandit/3.50","NICErsPRO","Microsoft URL Control - 5.01.4511","DittoSpyder","Foobot","SpankBot","BotALot","lwp-trivial/1.34","lwp-trivial","BunnySlippers","Microsoft URL Control - 6.00.8169","URLy Warning","Wget/1.6","Wget/1.5.3","Wget","LinkWalker","cosmos","moget","hloader","humanlinks","LinkextractorPro","Offline Explorer","Mata Hari","LexiBot","Web Image Collector","The Intraformant","True_Robot/1.0","True_Robot","BlowFish/1.0","JennyBot","MIIxpc/4.2","BuiltBotTough","ProPowerBot/2.14","BackDoorBot/1.0","toCrawl/UrlDispatcher","WebEnhancer","suzuran","TightTwatBot","VCI WebViewer VCI WebViewer Win32","VCI","Szukacz/1.4","QueryN Metasearch","Openfind data gatherer","Openfind","Xenu's Link Sleuth 1.1c"]}
{"domain":"hostgator.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"gcdn.co","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"aliexpress.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Googlebot","AdsBot-Google","AdsBot-Google-Mobile","*","YandexBot"]}
{"domain":"amplitude.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","cohere-ai","Bytespider","Amazonbot","Googlebot","Bingbot","DuckDuckBot"]}
{"domain":"name.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","MJ12bot"]}
{"domain":"adriver.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"akamaitech.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cloudns.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ietf.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mi.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"g.page","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"substack.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["BLEXBot","Twitterbot","*","facebookexternalhit"]}
{"domain":"mikrotik.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"amazonalexa.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"avsxappcaptiveportal.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","ENOTFOUND"]}
{"domain":"ipv4only.arpa","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"featureassets.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"mynetname.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nflxvideo.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"gamepass.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"mysql.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","msnbot"]}
{"domain":"quora.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":29,"stanza_tokens":["AASA-Bot","Amazonbot","Googlebot","Google-Extended","Facebot","Applebot","Twitterbot","LinkedInBot","Feedly","Pingdom","Applebot-Extended","Arquivo-web-crawler","Bytespider","Diffbot","GPTBot","ia_archiver","ImagesiftBot","JamesBOT","OAI-SearchBot","PerplexityBot","PetalBot","Scrapy","ChatGPT-User","Bravebot","CriteoBot/0.1","ClaudeBot","Claude-User","Claude-SearchBot","*"]}
{"domain":"goodreads.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":8,"stanza_tokens":["bingbot","EtaoSpider","GPTBot","CCBot","AmazonAdBot","facebookexternalhit","Mediapartners-Google","*"]}
{"domain":"eye4.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"netease.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",404]}
{"domain":"sohu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["Baiduspider","YisouSpider","Bytespider","msnbot","Googlebot","ToutiaoSpider","bingbot","DingTalkBot","Pinterestbot","sogou web spider","360Spider","Mediapartners-Google","*"]}
{"domain":"volcfcdndvs.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dotaplabs.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"addtoany.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","facebookexternalhit"]}
{"domain":"exp-tas.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"2mdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cookielaw.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"scribd.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"walmart.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Adsbot-Google","Mediapartners-Google","Slurp"]}
{"domain":"arxiv.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","Googlebot","Yahoo! Slurp","bingbot","Baiduspider","ToutiaoSpider","SQUID_configured_as_described_at_/help/faq/cache","YandexBot","Applebot","SemrushBot"]}
{"domain":"stackoverflow.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":418,"error":null,"attempts":[418,418]}
{"domain":"svc.ms","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"ioref.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"amzn.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"foxnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gandi-ns.fr","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"behance.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"docker.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"blogger.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"duolingo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Twitterbot","facebookexternalhit"]}
{"domain":"expireddomains.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"slideshare.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"beyondwickedmapping.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"shein.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"indiatimes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AmazonAdBot","meta-externalagent","*"]}
{"domain":"ft.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","CCBot","Google-Extended","GoogleOther","anthropic-ai","omgili","omgilibot","Bytespider","cohere-ai","magpie-crawler","NewsNow","PerplexityBot","Perplexity-User","Claude-Web","ClaudeBot","Diffbot","FacebookBot","Applebot-Extended","meta-externalagent","Meta-ExternalFetcher"]}
{"domain":"wired.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","FacebookExternalHit","Twitterbot","Pinterestbot","LinkedInBot","TikTokSpider","Storebot-Google","Google-InspectionTool","AmazonAdBot","Google-Extended","Google-CloudVertexBot","GoogleOther","Applebot-Extended","Amazonbot","meta-webindexer","meta-externalagent","meta-externalfetcher","ClaudeBot","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","cohere-training-data-crawler","cohere-ai","CCBot","PanguBot","PetalBot","Bytespider","Diffbot","DuckAssistBot","MistralAI-User","Timpibot","Webzio","Webzio-Extended","archive.org_bot","ia_archiver","ia_archiver-web.archive.org","heritrix"]}
{"domain":"atlassian.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["ia_archiver","*"]}
{"domain":"amazon.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"adobedtm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"skyhigh.cloud","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"paloaltonetworks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Twitterbot"]}
{"domain":"nic.direct","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"msedge.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"jsdelivr.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"nstld.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"360.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","AhrefsBot","SemrushBot","MJ12bot","DotBot"]}
{"domain":"google.com.hk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"xhamster.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"gosuslugi.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["SputnikBot","GoogleBot","*"]}
{"domain":"nease.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"vidaahub.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"run.app","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"mediatek.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"time.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"dynatrace.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jquery.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"firefox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"elasticbeanstalk.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ECONNRESET"]}
{"domain":"openx.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"sberbank.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"bdydns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"geobasket.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"uk.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"fandom.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"patreon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"dailymail.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":106,"stanza_tokens":["*","Googlebot-News","Mediapartners-Google","CrystalSemanticsBot","archive-it.org","archive.org_bot","ArchiveBot","Arquivo-web-crawler","Authory","babel","bl.uk_ldfc_bot","bl.uk_ldfc_renderbot","bne.es_bot","bnf.fr_bot","ecoresearch","ecoresearchCrawler","Europarchive.org","kb.dk_bot","Kulturarw3","LivelapBot","mediacloud","mirrorweb","MSIECrawler","nationalarchives.gov.uk","netarkivet.dk","netEstate NE Crawler","Nicecrawler","nl-israel_iaharvester2026","nlnbot","nlncrawler","PageFreezer","SafeDNS","SmarshBot","special_archiver","tkl.iis.u-tokyo.ac.jp","TurnitinBot","webarchiv.cz","WordPress.com mShots","XY-Archive-Compliance-Archiver","XY-Archive-Compliance-Crawler","AhrefsBot","AI2Bot","Ai2Bot-Dolma","Amazonbot","amazon-kendra-","anthropic-ai","Applebot-Extended","bedrockbot","Bytespider","CloudflareBrowserRenderingCrawler","CCBot","ChatGLM-Spider","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","Cotoyogi","DeepSeekBot","Diffbot","DuckAssistBot","EchoboxBot","FacebookBot","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","GoogleOther-Image","GoogleOther-Video","GPTBot","Grok","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","KunatoCrawler","LinerBot","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","MistralAI-User","OAI-Operator","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","Perplexity-User","PetalBot","QualifiedBot","Scrapy","Seekr","Sidetrade indexer bot","TaraGroup Intelligent Bot","TikTokSpider","Timpibot","VelenPublicWebCrawler"]}
{"domain":"tandfonline.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":5,"stanza_tokens":["*","facebookexternalhit","LinkedInBot","Twitterbot","GPTBot"]}
{"domain":"usatoday.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":287,"stanza_tokens":["anthropic-ai","AmazonBot","Claude","Claude-Web","claudeweb","Amzn-SearchBot","Amzn-User","AndiBot","cohere-ai","ClaudeBot","ChatGPT-User","Googlebot-News","GPTBot","CCBot","Google-Extended","DataForSeoBot","Diffbot","Diffbot-user","Kagibot","FacebookBot","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Scrapy","TurnitinBot","Applebot-Extended","Meta-ExternalAgent","meta-externalagent","OAI-SearchBot","Bytespider","Claude-User","Claude-SearchBot","AI2Bot","AhrefsBot","Ai2Bot-Dolma","Applebot","Arquivo-web-crawler","AspiegelBot","AwarioRssBot","AwarioSmartBot","Barkrowler","coccocbot-web","cohere-training-data-crawler","crawler4j","Crawlspace","dotbot","DuckAssistBot","FriendlyCrawler","Google-CloudVertexBot","https://hada.news","https://www.imediaethics.org","iaskspider/2.0","ICC-Crawler","icc-crawler","ICC Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","LivelapBot","MauiBot","Meltwater","Meta-ExternalFetcher","MJ12bot","Mojeek","MojeekBot","MoodleBot","NeticleBot","Netvibes","PanguBot","PetalBot","SeekportBot","Seekr","SemrushBot","SeznamBot","Sidetrade Indexer Bot","Timpibot","Turnitin","VelenPublicWebCrawler","Webzio-Extended","yacy","yacybot","Yandex","YandexAdditional","YandexAdditionalBot","YandexBot","YandexImages","YandexNews","Baiduspider","Baiduspider-image","BaiduSpider-render","Baiduspider-video","Sogou Web Spider","yisouspider"]}
{"domain":"t-mobile.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"netflix.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["*","googlebot","Googlebot-Video","Applebot","bingbot","Baiduspider","Baiduspider-mobile","Baiduspider-video","Baiduspider-image","NaverBot","Y!J-WSC","YJ-WSC","Yeti","Yandex","YandexBot","YandexMobileBot","YandexVideo","YandexWebmaster","YandexSitelinks","SeznamBot","facebookexternalhit","GPTBot","ChatGPT-User","OAI-SearchBot","Google-Extended","ClaudeBot","Claude-User","Claude-SearchBot","PerplexityBot","AdsBot-Google","Twitterbot","Adidxbot","Yahoo Pipes 1.0","Facebot"]}
{"domain":"icloud-content.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"aol.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":44,"stanza_tokens":["*","ADmantX","AlphaBot","anthropic-ai","AwarioRssBot","AwarioSmartBot","BLEXBot","Buzzbot","Bytespider","CCBot","ChatGPT-User","claritybot","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","huggingface","ImagesiftBot","img2dataset","magpie-crawler","Meltwater","Neevabot","news-please","NewsNow","Nutch","omgili","omgilibot","panscient.com","Perplexity-ai","PerplexityBot","PetalBot","PiplBot","scoop.it","Scrapy","Seekr","SentiBot","SeznamBot","TurnitinBot","YouBot","ZumBot"]}
{"domain":"live.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"visualstudio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cloudflareinsights.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"fast.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"deviantart.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Pinterestbot"]}
{"domain":"capcut.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":23,"stanza_tokens":["Baiduspider","360Spider","Yisouspider","Sogouspider","Googlebot","Applebot","Bingbot","DuckDuckBot","Naverbot","Twitterbot","Yandex","Facebot","facebookexternalhit","facebookcatalog","Google-Extended","GPTBot","ChatGPT-User","ClaudeBot","PerplexityBot","Applebot-Extended","MetaAI","CCBot","ugqaautomaticallybot"]}
{"domain":"firetvcaptiveportal.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","ETIMEDOUT"]}
{"domain":"browser-intake-datadoghq.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"impervadns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"oup.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"claude.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["ia_archiver","GPTBot","OAI-SearchBot","ChatGPT-User","*"]}
{"domain":"atlassian.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","atlassian-bot"]}
{"domain":"okta.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*AdobeEdgeOptimize/1.0*"]}
{"domain":"tripadvisor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":24,"stanza_tokens":["Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Cohere-ai","GPTBot","Google-CloudVertexBot","Img2dataset","PanguBot","PetalBot","SentiBot","YouBot","magpie-crawler","meta-externalagent","*","ChatGPT-User","ChatGPT-User/2.0","Gemini-Deep-Research","OAI-SearchBot","Google-Extended","applebot","PerplexityBot","bingbot"]}
{"domain":"trendmicro.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":24,"stanza_tokens":["Adsbot-Google","cludo","OAI-SearchBot","ChatGPT-User","ChatGPT-User/2.0","ClaudeBot","claude-web","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","Google-Extended","Googlebot","Bingbot","Amazonbot","Applebot","FacebookBot","meta-externalagent","LinkedInBot","Bytespider","DuckAssistBot","cohere-ai","CCBot","*"]}
{"domain":"health.mil","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cdngslb.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cornell.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com","Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com","HTML validator: Siteimprove_W3C_Validator/1.3","CSS Validator: Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0"]}
{"domain":"imgsmail.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wix.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"telegraph.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":59,"stanza_tokens":["*","endeca","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot-Extended","bedrockbot","Bytespider","CCBot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","Cotoyogi","DeepSeekBot","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","Grok","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","KunatoCrawler","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","MistralAI-User","OAI-Operator","OAI-SearchBot","omgili","omgilibot","PanguBot","Perplexity-User","PerplexityBot","PetalBot","QualifiedBot","Scrapy","Seekr","Sidetrade indexer bot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","WRTNBot"]}
{"domain":"360yield.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"palmplaystore.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"uol.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":50,"stanza_tokens":["*","Google-Extended","AI2Bot","Amazonbot","Anthropic-ai","Applebot","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","Cohere-ai","cohere-training-data-crawler","Cotoyogi","DataForSeoBot","Datenbank Crawler","DeepSeek","Devin","Diffbot","DuckAssistBot","FacebookBot","Factset_spyderbot","Gemini-Deep-Research","GoogleAgent-Mariner","ICC-Crawler","ImagesiftBot","Kangaroo Bot","Meta-ExternalAgent","Meta-ExternalFetcher","MistralAI-User","netEstate Imprint Crawler","NovaAct","omgili","Omgilibot","Operator","PanguBot","Peer39_crawler/1.0","Perplexity-User","PerplexityBot","PetalBot","Qwen","SemrushBot-OCOB","Timpibot","Twitterbot","Webzio-Extended","YouBot"]}
{"domain":"3lift.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"azure-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rlcdn.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"vedsalb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"berkeley.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ca.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eset.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","MJ12bot","BLEXBot"]}
{"domain":"teamviewer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"surveymonkey.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"deepl.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["AdsBot-Google","AdsBot-Google-Mobile","AdIdxBot","Mediapartners-Google","Ads-Naver","Y!J-BRY","Y!J-BRZ","Googlebot","Bingbot","Yeti","Y!J-ASR","Y!J-WSC","Slurp","*"]}
{"domain":"byteglb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tplinkcloud.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hotjar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mangosip.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"aboutads.info","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"hicloud.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"msidentity.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"t-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"e2ro.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"yahoodns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"loc.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["008","Baiduspider","Baiduspider-image","*"]}
{"domain":"grammarly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ieee.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":50,"stanza_tokens":["*","Facebot","facebookexternalhit","LinkedInBot","Twitterbot","GPTBot","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Crawlspace","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","PetalBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot"]}
{"domain":"mdpi.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"digitalocean.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oraclecloud.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"kwai.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["Baiduspider","Baiduspider-video","Googlebot","YoudaoBot","Sogou web spider","Sogou inst spider","360Spider","bingbot","*"]}
{"domain":"bidswitch.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ikea.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"klaviyo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["ClaudeBot","Claude-User","Claude-SearchBot","OAI-SearchBot","GPTBot","ChatGPT-User","PerplexityBot","Perplexity-User","Google-Agent","Google-NotebookLM","Gemini-Deep-Research","DuckAssistBot","YouBot","meta-externalagent","*"]}
{"domain":"inner-active.mobi","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"target.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tbcache.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"playstation.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AhrefsBot","YandexBot","bingbot","Algolia Crawler","PetalBot"]}
{"domain":"jotform.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":24,"stanza_tokens":["OAI-AdsBot","OAI-SearchBot","*","Yandexbot","LinkedInBot","Twitterbot","FacebookBot","FacebookExternalHit","Meta-ExternalAgent","Meta-ExternalFetcher","AdsBot-Google","AdsBot-Google-Mobile","AdIdxBot","Bingbot","GPTBot","ClaudeBot","Claude-Web","CCBot","Applebot-Extended","PerplexityBot","anthropic-ai","cohere-ai","Diffbot","Bytespider"]}
{"domain":"avito.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Yandex","Googlebot","*","Mediapartners-Google","YandexDirect"]}
{"domain":"pushy.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"my.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"adobedc.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"discord.media","fetch_ok":false,"usable":false,"reason":"other","status":520,"error":null,"attempts":[520,"ENOTFOUND",520]}
{"domain":"markmonitor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"att.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"accuweather.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"licdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"britannica.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":21,"stanza_tokens":["*","Mediapartners-Google","AdsBot-Google","AmazonAdBot","Bidswitchbot","CriteoBot","DoubleVerifyBot","DVBot","Grapeshot","Gumgum","IAS_admantx","IAS_crawler","IAS_wombles","Leikibot","Opebot-v","Peer39_crawler","Proximic","PubMatic Crawler Bot","SinceraSyntheticUser","SlickBot","TTD-Content"]}
{"domain":"googlezip.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"on.aws","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"timeweb.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UND_ERR_HEADERS_OVERFLOW","attempts":["UND_ERR_HEADERS_OVERFLOW","UND_ERR_HEADERS_OVERFLOW"]}
{"domain":"resolver.arpa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"comcast.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"googletagservices.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"cnet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":112,"stanza_tokens":["*","AddSearchBot","AI2Bot","Ai2Bot-DeepResearchEval","Ai2Bot-Dolma","Amazonbot","AmazonBuyForMe","Amzn-SearchBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot","Applebot-extended","atlassian-bot","AutoRAG","AwarioSmartBot","AzureAI-SearchBot","Big Sur AI","bigsur.ai","Brandwatch","Bravebot","Brightbot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","Datenbank-Crawler","DeepSeekBot","Devin","Diffbot","Direqt Anomura","DuckAssistBot","Echobot Bot","ExaBot","FacebookBot","Factset_spyderbot","FirecrawlAgent","GPTBot","iAsk","iAskBot","iaskspider","ICC Crawler","ICC-Crawler","ImageSiftBot","imageSpider","Kangaroo Bot","Kangaroo-Bot","KlaviyoAIBot","Kunato","laion-huggingface-processor","LCC","LINER Bot","LinerBot","LinkupBot","Manus-User","meta-externalagent","meta-externalfetcher","meta-webindexer","mistral.ai","MistralAI-User","netEstate Imprint Crawler","NovaAct","Novellum","Novellum AI Crawl","OAI-SearchBot","omgili","opencode","PanguBot","PeopleInc-DCipher-Scraper/1.1.0","Perplexity-User","PerplexityBot","PetalBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","SeekrBot","SemrushBot-OCOB","SemrushBotSwa","ShapBot","Spider","TavilyBot","TerraCotta","Timpibot","Trae","TwinAgent","Velen Crawler","VelenPublicWebCrawler","Webzio-Extended","Wrtn"]}
{"domain":"techcrunch.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":14,"stanza_tokens":["*","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","Google-Extended","GPTBot","omgili"]}
{"domain":"onetrust.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"threads.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["Amazonbot","Applebot-Extended","Brightbot","ClaudeBot","Google-Extended","GPTBot","PerplexityBot","PetalBot","Scrapy","uptimerobot","viberbot","YaK","Yandex","Yeti","Applebot","Bingbot","Discordbot","DuckDuckBot","facebookexternalhit","Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News","Google-InspectionTool","LinkedInBot","msnbot","Pinterestbot","Screaming Frog SEO Spider","seznambot","Slurp","teoma","TelegramBot","Twitterbot","*"]}
{"domain":"sagepub.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":42,"stanza_tokens":["*","Robozilla","Gigabot","Nutch","naverbot","yeti","Sogou web spider","YisouSpider","Applebot","Teoma","baiduspider","Googlebot","Googlebot-Image","Googlebot-Mobile","Googlebot-News","Googlebot-Video","Storebot-Google","Mediapartners-Google","AdsBot-Google","bingbot","msnbot","msnbot-media","adidxbot","Slurp","yahoo-mmcrawler","yandex","ia_archiver","Applebot-Extended","Google-Extended","GPTBot","ChatGPT-User","Omgilibot","Omgili","Diffbot","ImagesiftBot","cohere-ai","Amazonbot","Bytespider","CCBot","anthropic-ai","ClaudeBot","meta-externalagent"]}
{"domain":"redhat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sharethrough.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":33,"stanza_tokens":["*","YandexBot","Googlebot","Bingbot","Baiduspider","OAI-SearchBot","Sogou","DuckDuckBot","MojeekBot","SeznamBot","Qwantify","Yeti","Slurp","360Spider","meta-externalagent","ChatGPT-User","ClaudeBot","PerplexityBot","GPTBot","GoogleOther","Timpibot","SBIntuitionsBot","Bytespider","AI2Bot","Amazon Q","Claude-User","CCBot","Diffbot","YandexGPT","MistralBot","cohere-ai","youBot","NeevaAI"]}
{"domain":"bluehost.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"xiaomi.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"squarespace.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":30,"stanza_tokens":["AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","DuckAssistBot","FacebookBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","img2dataset","Meta-ExternalAgent","MyCentralAIScraperBot","omgili","omgilibot","Quora-Bot","TikTokSpider","YouBot","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","*"]}
{"domain":"braze.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"branch.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","GPTBot","CCBot","Google-Extended","PerplexityBot","ClaudeBot","anthropic-ai","Applebot-Extended"]}
{"domain":"bytedns1.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nike.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Baiduspider","HaoSouSpider","Sogou web spider","Sogou inst spider","Sogou spider2"]}
{"domain":"cbsnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":5,"stanza_tokens":["*","GPTBot","MAZBot","panscient.com","proximic"]}
{"domain":"elpais.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":200,"stanza_tokens":["PetalBot","bedrock-agentcore-browser","Amazon-Bot","Amazonbot","AmazonProductDiscoveryBot","Amzn-SearchBot","Amzn-User","Arquivo-web-crawler","bnf.fr_bot","Bytespider","Claude-Extension","ClaudeBot","Claude-User","Claude-SearchBot","Comet","CloudFlare-AlwaysOnline","Cloudflare-Healthchecks","CloudflareBrowserRendering","Cloudflare-Radar","CCBot","DeepSeekBot","DuckAssistBot","archive.org_bot","ia_archiver","MistralAI-User","Moreover","Adbeat","Argus","Baiduspider","Baiduspider-image","Bitdefender","Blogtrottr","BloombergBot","BoardReader","checkmy.ws","Clickagy","CookieBot","cXensebot","DataObserver","DomainStatsBot","Embedly","Feedspotbot","FlipboardProxy","Getintent","Google-Apps-Script","GoogleDrive","Grapevine6","Gumgum","HaloBot","HubSpot Crawler","IbouBot","Inout","LinerBot","MJ12bot","Manus","Meltwater","Metajob","Muckrack","NetSeer","Netvibes","Onyphe","OpenClaw","Opoint","OutbrainBot","PlumAnalytics","ProSitemaps","PerplexityBot","Perplexity-User","proximic","PubMatic","Quantcastbot","SearchConcepts","Semantic-Visions","SentiBot","Sindup","Sirdata","SmartologyBot","Superfeedr","surlybot","Synaptive","TinEye","Trendiction","TurnitinBot","Ubermetrics","Vebidoo","Webcorp","Weborama","WebPageTest","YandexAccessibilityBot","YandexDirect","YandexDirectDyn","YaDirectFetcher","YandexFavicons","YandexMetrika","YandexMobileBot","YandexNews","YandexRCA","YandexRenderResourcesBot","YandexUserproxy","*"]}
{"domain":"merriam-webster.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fontawesome.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cambridge.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":[],"path_restricted":["GPTBot","OAI-SearchBot"]}],"stanza_token_count":5,"stanza_tokens":["Yandex","ChatGPT-User","ChatGPT","*","Twitterbot"]}
{"domain":"dreamhost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ui-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"verisign.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"supertms.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"free.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":29,"stanza_tokens":["Googlebot","Googlebot-Image","Mediapartners-Google","Googlebot-News","Googlebot-Video","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","Storebot-Google","Bingbot","AdIdxBot","DuckDuckBot","Slurp","Qwantify","Pinterest","*","buenibot","EtaoSpider","Yandex","Baiduspider","Baiduspider-image","360Spider","MJ12bot","CazoodleBot","HaosouSpider","ia_archiver","Wget","TurnitinBot","HTTrack"]}
{"domain":"vkuser.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"imcmdb.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"rakuten.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AdsBot-Google","Googlebot"]}
{"domain":"easebar.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"withgoogle.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"independent.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Nutch"]}
{"domain":"amazon.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"amazon.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"nintendo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":25,"stanza_tokens":["*","Kangaroo Bot","Nutch","Omgili","Omgilibot","PanguBot","PetalBot","Timpibot","Webzio-Extended","AI2Bot","AI2Bot-Dolma","Bytespider","CCBot","Diffbot","FacebookBot","FriendlyCrawler","ICC-Crawler","GPTBot","Applebot-Extended","Claude-Web","Claudebot","cohere-ai","cohere-training-data-crawler","Google-Extended","Meta-ExternalAgent"]}
{"domain":"share.google","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"stackadapt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ozone.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"force.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"viber.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"service.gov.uk","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"vercel-dns-016.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"noaa.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"cdnvideo.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"optimizely.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["Amazonbot","Applebot-Extended","Brightbot","ClaudeBot","Google-Extended","GPTBot","PerplexityBot","PetalBot","Scrapy","uptimerobot","viberbot","YaK","Yandex","Yeti","Applebot","Bingbot","Discordbot","DuckDuckBot","facebookexternalhit","Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News","Google-InspectionTool","LinkedInBot","msnbot","Pinterestbot","Screaming Frog SEO Spider","seznambot","Slurp","teoma","TelegramBot","Twitterbot","*"]}
{"domain":"g.co","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":[400,404]}
{"domain":"squarespacedns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"conviva.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"speedtest.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AmazonAdBot","ia_archiver"]}
{"domain":"intercom.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mailchimp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"qlivecdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"quantserve.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Googlebot","Googlebot-Image","AdsBot-Google","GoogleOther","*"]}
{"domain":"unesco.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"dotomi.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ted.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":40,"stanza_tokens":["*","Baiduspider","OAI-SearchBot","ChatGPT-User","PerplexityBot","Perplexity-User","Claude-SearchBot","Claude-User","DuckAssistBot","MistralAI-User","YouBot","PetalBot","GPTBot","ClaudeBot","anthropic-ai","Claude-Web","CCBot","Google-Extended","Applebot-Extended","Amazonbot","meta-externalagent","Meta-ExternalAgent","FacebookBot","Bytespider","cohere-ai","cohere-training-data-crawler","PanguBot","Diffbot","FriendlyCrawler","ImagesiftBot","img2dataset","omgili","omgilibot","Webzio-Extended","Timpibot","Brightbot","DataForSeoBot","DeepSeekBot","iaskspider","GrokBot"]}
{"domain":"pinimg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":315,"stanza_tokens":["2checkout","AASA-Bot","AccessibleWebBot","Adagiobot","adbeat_bot","ADmantX","AdometryBot","Adsense-Snapshot-Google","AdsTxtCrawler","Adyen","AhrefsBot","AhrefsSiteAudit","Akismet","Algolia Crawler","AmazonProductDiscovery","Aolbot-News","APIs-Google","AppInsights","Applebot","AppleNewsBot","Archive-It","archive.org_bot","ArchiveBot","ArchiveTeam-ArchiveBot","AspiegelBot","Audisto Crawler","Awario","Baidu-ADS","Baiduspider","Barkrowler","bingbot","BingPreview","bitlybot","bl.uk_ldfc_bot","BLP_bbot","bordermanager","botify","BrandVerity","Bravebot","BrightEdge Crawler","BrightEdge-Crawler","BUbiNG","CaliperBot","CapitalOneBot","Catchpoint","cf-qualys-scanner","Chrome-Lighthouse","CJBot","ClarityBot","Clickagy Intelligence Bot v2","Cloudflare-AI-Search","CloudflareBrowserRenderingCrawler","coccocbot","ContextualBot","CookieReports","Coveobot","CriteoBot","curly-","cXensebot","Datadog","DataForSeoBot","Daum","demandbase","Discordbot","DotBot","DoubleVerifyBot","Doximity-Pipeline","Dragonbot","Dratabot","DuckAssistBot","DuckDuckBot","DuckDuckGo-Favicons-Bot","DVBot","eContext","ecurly Webhook","EngageBDR","europarchive.org","evidon","ExaleadCloudview","FacebookBot","facebookcatalog","facebookexternalhit","Facebot","FastDAST","FeedBurner","Feedfetcher-Google","FlipboardProxy","FlippBot","Freshping","GeedoProductSearch","Google Favicon","Google News","Google Page Speed Insights","Google Wireless Transcoder","Google-Apps-Script","Google-Certificates-Bridge","Google-CWS","Google-Display-Ads-Bot","Google-Firebase","Google-HotelAdsVerifier"]}
{"domain":"yelp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":31,"stanza_tokens":["Google-InspectionTool","Googlebot","Googlebot-Image","Googlebot-Mobile","Googlebot-Video","LinkedInBot","Twitterbot","facebookexternalhit","AdsBot-Google","BingPreview","Mediapartners-Google","STC-bot","Yahoo! Slurp","bingbot","OAI-SearchBot","ChatGPT-User","Claude-User","PerplexityUser","Perplexity-User","Claude-SearchBot","PerplexityBot","GPTBot","ClaudeBot","CCBot","Google-Extended","anthropic-ai","Claude-Web","Meta-ExternalAgent","Meta-ExternalFetcher","RuxitSynthetic","*"]}
{"domain":"ovh.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"amagi.tv","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","TimeoutError"]}
{"domain":"wbx2.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"twimg.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"lemonde.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":286,"stanza_tokens":["*","Googlebot-Image","Googlebot-News","Cision","Talkwater","Jetbot","kbcrawl","Newzbin","Qwam content intelligence","Youmag","Synthesio","trendybuzz","scoop.it","linkfluence","grub-client","ia_archiver-web.archive.org","k2spider","libwww","wget","5erue","adequat","adequat-systems","coexel","leadbox","mention","mytwip","opinion-tracker","proxem","score3","vecteurplus","verticalsearch","vsw","winello","Fetch","infoseek","MSIECrawler","Offline Explorer","sitecheck.internetseer.com","Teleport","TeleportPro","WebCopier","WebStripper","Zealbot","asknread.com","omgilibot","omgili","CCBot","Google-Extended","anthropic-ai","Claude-Web","ClaudeBot","Applebot-Extended","Webzio-Extended","Amazonbot","Timpibot","AI2Bot","cohere-training-data-crawler","DuckAssistBot","Kangaroo Bot","PanguBot","MistralAI-User","facebookbot","FacebookBot","5emeRue","ACQUIRE MEDIA","ACTIV Financial (CME Group)","AlphaSense","AmiSoftware","archive.org_bot","Archive-It","ArgClrInt","Ask n read","Augure","auramundi","AwarioRssBot","AwarioSmartBot","Barchart.com","BattleFin","Bernin IT","Blackboard Safeassign","BLP_bbot","bluematrix","Brandwatch","Briefcase.news","Buck","Bytespider","CCBo","CikisiBot","Coexel","cohere-ai","Comtex News Network","ConveraCrawler","Copyright Licensing Agency","Corporama","D&B Hoovers","Data Expression","Data Observer","Dataminr","Dealogic","Diffbot"]}
{"domain":"outbrain.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zdnscloud.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"1rx.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"smaato.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sc-cdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"anydesk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"agkn.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"spiegel.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":38,"stanza_tokens":["*","emetriqContextualBot","Mozilla/5.0 (compatible; OGDWCtxCrawler)","AmazonAdBot","OAI-SearchBot","ChatGPT-User","cohere-ai","cohere-training-data-crawler","Webzio-Extended","YouBot","GPTBot","Applebot-Extended","CCBot","magpie-crawler","ia_archiver","omgili","omgilibot","Baiduspider","AhrefsBot","DataForSeoBot","Yeti","SemrushBot","sentibot","MJ12bot","Bytespider","SirdataBot","LCC","TurnitinBot","BLEXBot","dotbot","ImagesiftBot","anthropic-ai","Claude-Web","ClaudeBot","Timpibot","Meta-ExternalAgent","FacebookBot","Diffbot"]}
{"domain":"creativecdn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hihonorcloud.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"a-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bx-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"samsungacr.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zillow.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":19,"stanza_tokens":["*","facebookexternalhit","LinkedInBot","Twitterbot","YouBot","YandexAdditional","YandexAdditionalBot","Applebot-Extended","Amazonbot","amazon-kendra","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","meta-externalagent","Meta-ExternalAgent","ActiveComplyBot","CriteoBot","GumGumBot","TaraGroup Intelligent Bot"]}
{"domain":"google.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"consultant.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ovscdns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"lijit.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"moe.video","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","GoogleBot","Yandex"]}
{"domain":"mediafire.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["ia_archiver","*"]}
{"domain":"amazon.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":102,"stanza_tokens":["*","EtaoSpider","Twitterbot","AmazonAdBot","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok"]}
{"domain":"dyingbirds.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"seznam.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","SeznamBot","SeznamSocialBot","Twitterbot"]}
{"domain":"ya.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Bingbot","Yandex","Googlebot","YandexAdditional"]}
{"domain":"archive-it.group","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mhverifier.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",403]}
{"domain":"wps.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bidr.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nel.goog","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cursor.sh","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kick.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"allaboutcookies.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bandcamp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":21,"stanza_tokens":["*","NextGenSearchBot","GeedoShopProductFinder","ImagesiftBot","EdisterBot","Ezooms","SWEBot","discobot","SemrushBot","grapeshot","BUbiNG","Bytespider","CCBot","ClaudeBot","Diffbot","FacebookBot","meta-externalagent","Google-Extended","GPTBot","omgili","Amazonbot"]}
{"domain":"newrelic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Swiftbot","phantomjs","phantomJS","DatadogSynthetics"]}
{"domain":"stripchat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nbcnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":37,"stanza_tokens":["*","Twitterbot","Amazonbot","AmazonBot","anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","Claude-User","Claude-SearchBot","cohere-ai","DataForSeoBot","Diffbot","FacebookBot","Google-Extended","GPTBot","magpie-crawler","Meta-ExternalAgent","meta-externalagent","MistralAI-User","NagetBot","NewsNow","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Perplexity-User","Scrapy","TurnitinBot"]}
{"domain":"amazon.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"dbankcloud.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"contentsquare.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"criteo.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"networkadvertising.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"asus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"homedepot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"openstreetmap.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wattpad.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":3,"stanza_tokens":["*","RankLite","GPTBot"]}
{"domain":"sophos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":22,"stanza_tokens":["Anthropic-ai","ClaudeBot","Bingbot","BingAI","BingPreview","ChatGPT-User","CopilotBot","Googlebot","Google-Extended","GPTBot","LlamaBot","OAI-SearchBot","PerplexityBot","xAI-Bot","Amazonbot","FirecrawlAgent","AndiBot","ExaBot","PhindBot","YouBot","PetalBot","DeepSeekBot"]}
{"domain":"wb.ru","fetch_ok":false,"usable":false,"reason":"other","status":498,"error":null,"attempts":[498,498]}
{"domain":"steamstatic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"amazon.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"ks-cdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"python.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["HTTrack","puf","MSIECrawler","Krugle","Nutch","*"]}
{"domain":"google.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"id5-sync.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"bamgrid.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"shopee.co.id","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot","Googlebot-Mobile","Bingbot","*"]}
{"domain":"yieldmo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iso.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bitrix24.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"wp.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":7,"stanza_tokens":["*","GPTBot","CCBot","PerplexityBot","Perplexity-User","OAI-SearchBot","ChatGPT-User"]}
{"domain":"mercadolivre.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User"],"open":[],"path_restricted":["Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":13,"stanza_tokens":["Amazonbot","PerplexityBot","Perplexity-User","ClaudeBot","Claude-User","GPTBot","ChatGPT-User","FacebookExternalHit","FacebookBot","Twitterbot","LinkedInBot","Bingbot","*"]}
{"domain":"wswebcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"dyndns.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"b-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"imrworldwide.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"adguard-vpn.online","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"safebrowsing.apple","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"anthropic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"marriott.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Baiduspider","*"]}
{"domain":"box.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"it.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"zoom.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["*","Googlebot","LinkedInBot","facebookexternalhit","Twitterbot","PinterestBot","InstagramBot","Slackbot","WhatsApp","redditbot","Tumblr","TikTokBot","Snapchat"]}
{"domain":"ring.com","fetch_ok":false,"usable":false,"reason":"other","status":406,"error":null,"attempts":[406,406]}
{"domain":"adform.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Twitterbot","GoogleOther","*"]}
{"domain":"bugsnag.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"prnewswire.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uber.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AdsBot-Google","Googlebot-Image"]}
{"domain":"allegro.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Mediapartners-Google"]}
{"domain":"herokuapp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hbr.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":7,"stanza_tokens":["*","008","FeedBurner/1.0","GPTBot","CCBot","ChatGPT-User","Google-Extended"]}
{"domain":"vercel-dns-017.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"nikkei.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"appcenter.ms","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"mlb.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"worldnic.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"rt.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"wbbasket.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bunnyinfra.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fda.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["vspider","usasearch","*"]}
{"domain":"wpguardian.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gumgum.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["GPTBot","ChatGPT-User","OAI-SearchBot","anthropic-ai","ClaudeBot","GoogleOther","Google-Extended","Googlebot","Googlebot-Image","Applebot","Bingbot","PerplexityBot","ScreamingFrogCrawler","Brightbot","*"]}
{"domain":"nmrodam.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"washington.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"teads.tv","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"dnspod.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"tencent.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wpguardian.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"apnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":14,"stanza_tokens":["*","CCBot","GPTBot","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","PerplexityBot","Amazonbot","Applebot-Extended","Bytedance","Timpibot","Claude-User","Claude-SearchBot"]}
{"domain":"fiverr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AdsBot-Google"]}
{"domain":"theverge.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":["GPTBot"],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":105,"stanza_tokens":["Amazonbot","Applebot","GPTBot","A6-Indexer","AhrefsBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot-Extended","Baiduspider","betaBot","BLEXBot","botify","brands-bot-logo","Brightbot","Bytespider","calculon","CCBot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","collection@infegy.com","Cotoyogi","Crawl4AI","CrunchBot","deepcrawl","DeepSeekBot","Diffbot","DirBuster","DTS","DuckAssistBot","EasouSpider","FacebookBot","FirecrawlAgent","FriendlyCrawler","GlutenCrawler","Google-CloudVertexBot","Google-Extended","GrapeshotCrawler","iaskspider","ICC-Crawler","ImagesiftBot","img2dataset","JackCrawl","Jetslide","linkapedia_bot","Mediatoolkitbot","MegaIndex","meta-externalagent","meta-externalfetcher","Microsoft BITS","Microsoft URL Control","Microsoft-WebDAV-MiniRedir","MistralAI-User","news-please","Novellum","NTENTbot","Nutch","omgili","Omgilibot","OpenLinkProfiler","Perplexity-User","PerplexityBot","PetalBot","Primalbot","python-requests","Re-Animator","Riddler","SafeDNSBot","SBIntuitionsBot","Scrapy","SearchAtlas","SemanticScholarBot","SemrushBot","SentiBot","Sogou","Spider-Pig","sqlmap","StorygizeBot","Test Certificate Info","TikTokSpider","Timpibot","Turnitin","uipbot","vBSEO","VegeBot","VelenPublicWebCrawler","Web Link Validator","WebCruiser"]}
{"domain":"imgur.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":[429,429,429]}
{"domain":"sedo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Riddler"]}
{"domain":"otto.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","bingbot","GPTBot","ChatGPT-User","OAI-Searchbot","PerplexityBot","Perplexity-User"]}
{"domain":"cdn-vk.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"perplexity.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"liadm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ameblo.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Googlebot","*","bingbot"]}
{"domain":"people.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"shopee.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot","Googlebot-Mobile","Bingbot","*"]}
{"domain":"shop.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ln-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"alipaydns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"rakuten.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","naverbot","yeti","grapeshot"]}
{"domain":"cmediahub.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"primevideo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":[],"path_restricted":["GPTBot","OAI-SearchBot"]},{"family":"anthropic","blocked":["Claude-User"],"open":[],"path_restricted":["ClaudeBot","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":94,"stanza_tokens":["*","AddSearchBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","Andibot","Anomura","atlassian-bot","Brightbot","BuddyBot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT-User","Claude-User","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-ai","cohere-training-data-crawler","Copilot","CopilotNative","CopilotSapphire","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","EtaoSpider","factset_spyderbot","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Firebase","Google-NotebookLM","GoogleAgent-Mariner","GoogleAgent-Shopping","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","iaskspider","IbouBot","ICC-Crawler","img2dataset","ImagesiftBot","imageSpider","imgproxy","ISSCyberRiskCrawler","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-externalagent","meta-externalfetcher","meta-webindexer","MistralAI-User","netEstate Imprint Crawler","omgili","Panscient","panscient.com","PanguBot","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","Scrapy","SemrushBot-SWA","ShapBot","Sidetrade indexer bot","TavilyBot","TerraCotta","Thinkbot","Timpibot","TwinAgent","VelenPublicWebCrawler","webzio-extended","wpbot","WRTNBot","xAI-Grok","YaK","YouBot","ZanistaBot"]}
{"domain":"whecloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"myspace.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_VERIFY_LEAF_SIGNATURE","attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE","UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"turn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mega.co.nz","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"lenovo.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"globalsign.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","Claude-SearchBot","PerplexityBot","GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot"]}
{"domain":"appspot.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"disneyplus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":87,"stanza_tokens":["*","Googlebot","Googlebot-Video","Googlebot-Image","Googlebot-News","Storebot-Google","Google-InspectionTool","Applebot","Bingbot","BingPreview","Slurp","DuckDuckBot","Baiduspider","Baiduspider-mobile","Baiduspider-video","Baiduspider-image","NaverBot","Y!J-WSC","YJ-WSC","Yeti","Yandex","YandexBot","YandexMobileBot","YandexVideo","YandexWebmaster","YandexSitelinks","SeznamBot","PetalBot","Sogou","Qwantify","Ecosia","360Spider","Coc Coc","Qwantbot","Botify","SemrushBot","Screaming Frog SEO Spider","DataForSeoBot","ia_archiver","Facebot","Twitterbot","facebookexternalhit","instagram","Threads","TikTok","Pinterest","Pinterestbot","redditbot","WhatsApp","Snapchat","TelegramBot","Viber","KakaoTalk","Line","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","Mediapartners-Google","Google-Ads-Creatives-Assistant","FacebookAdPreview","meta-externalads","Adidxbot","Bytespider","IAS-Crawler","DoubleVerifyBot","Moatbot","TheTradeDesk","TTDCrawler","CriteoBot","GPTBot","ChatGPT-User","OAI-SearchBot","Google-Extended","ClaudeBot","Claude-User","Claude-SearchBot","PerplexityBot","Applebot-Extended","Meta-ExternalAgent","Meta-ExternalFetcher","Amazonbot","CCBot","Google-Agent","GrokBot","YouBot","DuckAssistBot","DeepSeekBot"]}
{"domain":"airbnb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["Googlebot","Bingbot","YandexBot","Baiduspider","OAI-Searchbot","ChatGPT-User","GPTBot","meta-externalagent","PerplexityBot","anthropic-ai","ClaudeBot","cohere-ai","ImagesiftBot","*"]}
{"domain":"grammarly.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tplinknbu.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"linode.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"bild.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":84,"stanza_tokens":["AhrefsBot","AI2Bot","Ai2Bot-Dolma","aiHitBot","Andibot","Applebot-Extended","Awario","Baiduspider","bedrockbot","BLEXBot","Brightbot 1.0","Bytespider","CCBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","DataForSeoBot","Datenbank Crawler","Devin","Diffbot","dotbot","DuckAssistBot","Echobot Bot","EchoboxBot","Factset_spyderbot","FirecrawlAgent","Flamingo_SearchEngine","FriendlyCrawler","ia_archiver","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","LCC","magpie-crawler","MistralAI-User","MistralAI-User/1.0","MJ12bot","MyCentralAIScraperBot","netEstate Imprint Crawler","NovaAct","omgili","omgilibot","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot","Scrapy","SemrushBot","SemrushBot-OCOB","SemrushBot-SA","SemrushBot-SWA","sentibot","Sidetrade indexer bot","SirdataBot","sogou spider","SummalyBot","TerraCotta","Thinkbot","TikTokSpider","Timpibot","TurnitinBot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","wpbot","YandexAdditional","YandexAdditionalBot","Yeti","YouBot","Googlebot-News","*"]}
{"domain":"moloco.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":68,"stanza_tokens":["*","Teleport","WebZIP","CherryPicker","ERNIE-Bot","BlowFish/1.0","grub","Bytespider","SiteSnagger","Titan","Barkrowler","008","Alexibot","BackDoorBot/1.0","Bullseye/1.0","ExtractorPro","Kenjin Spider","WWW-Collector-E","Sogou","Yandex","Mail.RU_Bot","MegaIndex","Szukacz/1.4","BotALot","BunnySlippers","BuiltBotTough","CheeseBot","Crescent","EroCrawler","Foobot","Harvest/1.5","InfoNaviRobot","JennyBot","Mister PiX","moget","NetAnts","NICErsPRO","ProPowerBot/2.14","Radiation Retriever","RepoMonkey","RepoMonkey Bait & Tackle/v1.01","True_Robot","URL_Spider_Pro","Web Image Collector","WebAuto","WebBandit","WebCopier","Website Quester","Webster Pro","WebStripper","Wget","Zeus","dotbot","Rogerbot","ia_archiver","MJ12bot","Xenu","AhrefsBot","CCBot","SemrushBot","DataForSeoBot","XoviBot","Screaming Frog SEO Spider","Keyword Density/0.9","LinkextractorPro","LinkScan/8.1a Unix","BLEXbot","spbot"]}
{"domain":"princeton.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"33across.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mixpanel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lefigaro.fr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"zoho.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot-Image","zs-crawler","zoho-scrapy"]}
{"domain":"academia.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":59,"stanza_tokens":["Googlebot","APIs-Google","Mediapartners-Google","msnbot","bingbot","BingPreview","MicrosoftPreview","Slurp","DuckDuckbot","baiduspider","seznambot","naverbot","teoma","Yandex","ScoutJet","GurujiBot","Exabot","Solbot","FAST","VoilaBot","daumoa","ArchitextSpider","Twitterbot","SocSciBot","ia_archiver","coccoc","Browsershots","SiteArc","Cliqzbot","proximic","OnCrawlBot","Botify","Googlebot-Image","Googlebot-Video","Googlebot-News","Google-InspectionTool","Gemini-Deep-Research","OAI-SearchBot","ChatGPT-User","PerplexityBot","Perplexity‑User","Claude-SearchBot","Claude-User","DuckAssistBot","facebookexternalhit","LinkedInBot","Bluesky","Discordbot","TelegramBot","Slackbot","Applebot","YouBot","MistralAI-User","Kagibot","TavilyBot","Applebot-Extended","GPTBot","Google-Extended","*"]}
{"domain":"liftoff.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"readthedocs.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"usps.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"daum.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["GoogleOther","*"]}
{"domain":"disqus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pexels.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":["anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":16,"stanza_tokens":["*","AdsBot-Google","AdsIdxBot","Google-InspectionTool","facebookexternalhit","Claude-SearchBot","Claude-User","Perplexity-User","Applebot-Extended","GPTBot","Google-Extended","Bytespider","ClaudeBot","CCBot","PerplexityBot","Twitterbot"]}
{"domain":"thenai.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nflximg.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","SELF_SIGNED_CERT_IN_CHAIN","TimeoutError"]}
{"domain":"lgtvcommon.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ddnss.de","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"blackberry.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Uptimebot"]}
{"domain":"discordapp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Twitterbot","nsa"]}
{"domain":"hstgr.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"erome.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"onetag-sys.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"no-ip.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"myqcloud.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"afilias-nst.info","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"tencent-cloud.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"list-manage.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"a-mo.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"UND_ERR_CONNECT_TIMEOUT"]}
{"domain":"hostgator.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mayoclinic.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ssl-images-amazon.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"dbankcloud.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"npr.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"githubusercontent.com","fetch_ok":false,"usable":false,"reason":"other","status":500,"error":null,"attempts":["ENOTFOUND",500,"TimeoutError"]}
{"domain":"mcafee.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"samsungapps.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"adjust.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":[429,429,"TimeoutError"]}
{"domain":"2gis.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"spaceweb.pro","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"yellowblue.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","UND_ERR_CONNECT_TIMEOUT"]}
{"domain":"alicdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ups.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nic.network","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"rackspace.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"typekit.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"crwdcntrl.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"telephony.goog","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"mckinsey.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"dtkn.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"kunluncan.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"360safe.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","UND_ERR_CONNECT_TIMEOUT","TimeoutError"]}
{"domain":"chinamobile.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"itch.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"qualtrics.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hath.network","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"afilias-nst.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"spectrum.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"amazon.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"onlyfans.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"spotifycdn.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"livejournal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Yandex","spbot","AhrefsBot","*"]}
{"domain":"netcraze.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xvideos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"welt.de","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"orderbox-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"change.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"chess.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"discogs.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"unpkg.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"figma.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["anthropic-ai"],"open":["Claude-SearchBot"],"path_restricted":["ClaudeBot","Claude-User"]}],"stanza_token_count":24,"stanza_tokens":["Fasterfox","Nutch","spock","OmniExplorer_Bot","MJ12bot","TurnitinBot","BecomeBot","genieBot","MLBot","GPTBot","cohere-ai","anthropic-ai","CCBot","FacebookBot","Google-Extended","omgilibot","omgili","GleanSearchCrawlerBot","amazon-kendra","*","CHATGPT-User","OAI-SearchBot","PerplexityBot","Claude-SearchBot"]}
{"domain":"herokudns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ebay.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":15,"stanza_tokens":["*","bingbot","AdsBot-Google","AdIdxBot","Twitterbot","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","PerplexityBot","anthropic-ai","AmazonBot","meta-externalagent","GPTBot","Applebot-Extended"]}
{"domain":"synology.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ndtv.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"scdn.co","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"ipinfo.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pornhub.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":17,"stanza_tokens":["*","Twitterbot","Amazonbot","Anthropic-ai","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","Cohere-ai","DataForSeoBot","ImagesiftBot","Magpie-crawler","Omgili","Omgilibot","Peer39_crawler","Peer39_crawler/1.0","YouBot"]}
{"domain":"ys7.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eporner.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AhrefsBot","*"]}
{"domain":"cdnhwc1.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","CERT_HAS_EXPIRED"]}
{"domain":"usgovcloudapi.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"www.gov.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","*"]}
{"domain":"dailymail.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":106,"stanza_tokens":["*","Googlebot-News","Mediapartners-Google","CrystalSemanticsBot","archive-it.org","archive.org_bot","ArchiveBot","Arquivo-web-crawler","Authory","babel","bl.uk_ldfc_bot","bl.uk_ldfc_renderbot","bne.es_bot","bnf.fr_bot","ecoresearch","ecoresearchCrawler","Europarchive.org","kb.dk_bot","Kulturarw3","LivelapBot","mediacloud","mirrorweb","MSIECrawler","nationalarchives.gov.uk","netarkivet.dk","netEstate NE Crawler","Nicecrawler","nl-israel_iaharvester2026","nlnbot","nlncrawler","PageFreezer","SafeDNS","SmarshBot","special_archiver","tkl.iis.u-tokyo.ac.jp","TurnitinBot","webarchiv.cz","WordPress.com mShots","XY-Archive-Compliance-Archiver","XY-Archive-Compliance-Crawler","AhrefsBot","AI2Bot","Ai2Bot-Dolma","Amazonbot","amazon-kendra-","anthropic-ai","Applebot-Extended","bedrockbot","Bytespider","CloudflareBrowserRenderingCrawler","CCBot","ChatGLM-Spider","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","Cotoyogi","DeepSeekBot","Diffbot","DuckAssistBot","EchoboxBot","FacebookBot","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","GoogleOther-Image","GoogleOther-Video","GPTBot","Grok","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","KunatoCrawler","LinerBot","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","MistralAI-User","OAI-Operator","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","Perplexity-User","PetalBot","QualifiedBot","Scrapy","Seekr","Sidetrade indexer bot","TaraGroup Intelligent Bot","TikTokSpider","Timpibot","VelenPublicWebCrawler"]}
{"domain":"shopeemobile.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","UND_ERR_CONNECT_TIMEOUT",200]}
{"domain":"goskope.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"azure-devices.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"eu.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"fbsbx.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"bankofamerica.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","gsa-crawler","OmniExplorer_Bot","Googlebot-Mobile","msnbot-mobile","YahooSeeker/M1A1-R2D2"]}
{"domain":"dv.tech","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ENOTFOUND"]}
{"domain":"tiktokw.us","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"deloitte.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"arcgis.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"usda.gov","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"healthline.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":163,"stanza_tokens":["*","Mediapartners-Google","Adsbot-Google","AmazonAdbot","Amazonbot","Amzn-SearchBot","Amzn-User","amazon-kendra","amazon-QBusiness","bedrockbot","aws-quick-on-behalf-of-","AmazonProductDiscoverybot","AmazonBuyForMe","AmazonSellerInitiatedListing","NovaAct","Twitterbot","NimbleCrawler","grapeshot","BotRightHere","WebZip","larbin","b2w/0.1","Copernic","psbot","Python-urllib","URL_Spider_Pro","CherryPicker","EmailCollector","EmailSiphon","WebBandit","EmailWolf","ExtractorPro","CopyRightCheck","Crescent","SiteSnagger","ProWebWalker","CheeseBot","LNSpiderguy","Alexibot","Teleport","TeleportPro","MIIxpc","Telesoft","Website Quester","moget/2.1","WebZip/4.0","WebStripper","WebSauger","WebCopier","NetAnts","Mister PiX","WebAuto","TheNomad","WWW-Collector-E","RMA","libWeb/clsHTTP","asterias","httplib","turingos","spanner","InfoNaviRobot","Harvest/1.5","Bullseye/1.0","Mozilla/4.0 (compatible; BullsEye; Windows 95)","Crescent Internet ToolPak HTTP OLE Control v.1.0","CherryPickerSE/1.0","CherryPickerElite/1.0","WebBandit/3.50","NICErsPRO","Microsoft URL Control - 5.01.4511","DittoSpyder","Foobot","SpankBot","BotALot","lwp-trivial/1.34","lwp-trivial","BunnySlippers","Microsoft URL Control - 6.00.8169","URLy Warning","Wget/1.6","Wget/1.5.3","Wget","LinkWalker","cosmos","moget","hloader","humanlinks","LinkextractorPro","Offline Explorer","Mata Hari","LexiBot","Web Image Collector","The Intraformant","True_Robot/1.0","True_Robot","BlowFish/1.0","JennyBot","MIIxpc/4.2","BuiltBotTough","ProPowerBot/2.14"]}
{"domain":"flashtalking.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"richaudience.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yandexcloud.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"go-mpulse.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"state.gov","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TypeError","attempts":["TypeError","TypeError"]}
{"domain":"jimdo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":36,"stanza_tokens":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","Claude-Web","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","Gemini-Deep-Research","Google-CloudVertexBot","Applebot-Extended","DuckAssistBot","MistralAI-User","cohere-ai","cohere-training-data-crawler","CCBot","FacebookBot","meta-externalagent","Meta-ExternalFetcher","Amazonbot","YouBot","Webzio","Webzio-Extended","omgili","Omgilibot","Diffbot","Bytespider","AI2Bot","Kangaroo Bot","PanguBot","PetalBot","SemrushBot-OCOB","*"]}
{"domain":"jetbrains.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","Baiduspider"]}
{"domain":"akahost.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mailchi.mp","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Twitterbot","facebookexternalhit","LinkedInBot","*"]}
{"domain":"cloudinary.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["008","Semrushbot-SA","Semrushbot-SI","*"]}
{"domain":"starlink.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nintendo.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"latimes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":13,"stanza_tokens":["*","Googlebot-News","Twitterbot","Mediapartners-Google","AmazonAdBot","Bytespider","ClaudeBot","Diffbot","discobot","GPTBot","ImagesiftBot","omgili","omgilibot"]}
{"domain":"genius.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"amazon.com.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"epa.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"smartthings.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"biblegateway.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kahoot.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["Googlebot","*","facebookexternalhit","slackbot-linkexpanding","slack-imgproxy","twitterbot","whatsapp","skypeuripreview","bf-directline","LinkedInBot","snippet","wakeletlinkexpander"]}
{"domain":"eeroup.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"blackhub.team","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"me.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"focus.de","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"themeforest.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Pinterest","CCBot"]}
{"domain":"poki.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AmazonAdBot","*"]}
{"domain":"ebay.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":19,"stanza_tokens":["*","bingbot","AdsBot-Google","AdIdxBot","Twitterbot","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","PerplexityBot","anthropic-ai","AmazonBot","meta-externalagent","GPTBot","Applebot-Extended","OAI-SearchBot","ChatGPT-User","Claude-SearchBot","Claude-User"]}
{"domain":"nationalgeographic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fidelity.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["IDentity","AdsBot-Google","*"]}
{"domain":"freepik.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Mediapartners-Google","Adsbot-Google","Twitterbot"]}
{"domain":"vivo.com.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"life360.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"umich.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"coupang.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"samsungosp.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"mercadolibre.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User"],"open":[],"path_restricted":["Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":13,"stanza_tokens":["Amazonbot","PerplexityBot","Perplexity-User","ClaudeBot","Claude-User","GPTBot","ChatGPT-User","FacebookExternalHit","FacebookBot","Twitterbot","LinkedInBot","Bingbot","*"]}
{"domain":"keenetic.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"seedtag.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"abovedomains.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"youku.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"character.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"chaturbate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"usercontent.goog","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"jd.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":21,"stanza_tokens":["*","Googlebot","Googlebot-Image","Googlebot-Mobile","Googlebot-Video","Googlebot-News","Bingbot","msnbot","bingbot-mobile","Baiduspider","Baiduspider-image","Baiduspider-video","YandexBot","YandexBot-Image","YandexBot-Mobile","Sogou web spider","Sogou Pic Spider","360Spider","HaosouSpider","Sosoimagespider","Sosospider"]}
{"domain":"worldbank.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"recaptcha.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rutube.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Yandex","Googlebot","*","Bingbot"]}
{"domain":"kontur.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bytefcdn-ttpus.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bytefcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"theatlantic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":45,"stanza_tokens":["*","AmazonAdBot","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","ArcSpanScraper","AwarioRssBot","AwarioSmartBot","BingBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","DuckAssistBot","FacebookBot","Google-Extended","Googlebot","GPTBot","magpie-crawler","Meta-ExternalAgent","Meta-ExternalFetcher","MistralAI-User","NewsNow","news-please","omgili","omgilibot","OAI-SearchBot","Operator","PetalBot","PerplexityBot","Perplexity-User","ProRataInc","Quora-Bot","Scrapy","TerraCotta","TimpiBot","TurnitinBot","archive.org_bot"]}
{"domain":"agora.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ubnt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"translate.goog","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"weforum.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"webmd.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":7,"stanza_tokens":["*","AdsBot-Google","CCBot","ChatGPT-User","ClaudeBot","Doximity","GPTBot"]}
{"domain":"px-cloud.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ci-servers.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ryanair.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"kickstarter.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"adgrx.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"mozilla.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lowes.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"repubblica.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":109,"stanza_tokens":["*","Googlebot-News","GPTBot","Google-Extended","CCBot","anthropic-ai","Omgilibot","FacebookBot","cohere-ai","PetalBot","Yandex","PerplexityBot","Amazonbot","AhrefsBot","archive.org_bot","ArchiveBot","Arquivo-web-crawler","AwarioRssBot","AwarioSmartBot","BacklinkCrawler","BIXOCRAWLER","BLEXBot","Bytespider","Cliqzbot","ConveraCrawler","crawler4j","CrystalSemanticsBot","DataForSeoBot","Diffbot","discobot","discoverybot","dloader(NaverRobot)","dotbot","Download Ninja","DuckAssistBot","europarchive.org","Exabot","ExtractorPro","Fetch","Flamingo_SearchEngine","FriendlyCrawler","Gigabot","GumGum Bot","ia_archiver","ia_archiver-web.archive.org","ImagesiftBot","IstellaBot","Jetbot","JikeSpider","Jyxobot","k2spider","Kangaroo Bot","kbcrawl","LexxeBot/1.0","LinkArchiver","LinkextractorPro","Livelapbot","magpie-crawler","meta-externalfetcher","MJ12bot","Moreoverbot","MSIECrawler","nabot","NaverBot","NerdByNature.Bot","netEstate NE Crawler","nNetSeer crawler","NextGenSearchBot","Nicecrawler","NPBot","nutch","Openbot","PanguBot","peer39_crawler","peer39_crawler/1.0","PiplBot","Pixray-Seeker","Primalbot","psbot","QuerySeekerSpider","Quora-Bot","rogerbot","Scrapy","searchpreview","SemrushBot","SEOENGBot","SEOkicks-Robot","SiteBot","SiteBot/0.1","Slurp","Sosospider","spbot","Timpibot","trendictionbot","trovitBot","True_Robot","TurnitinBot","UbiCrawler","uMBot-LN","URL_Spider_Pro"]}
{"domain":"docker.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"odoo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["PetalBot","*"]}
{"domain":"ci-servers.org","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tsyndicate.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"msftauth.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"giphy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","PetalBot"]}
{"domain":"whitehouse.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vercel.app","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"theconversation.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":152,"stanza_tokens":["*","bingbot","AddSearchBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","AliyunSecBot/Aliyun","amazon-kendra","Amazonbot","AmazonBuyForMe","Amzn-SearchBot","Amzn-User","Andibot","Anomura","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot","Applebot-Extended","Aranet-SearchBot","atlassian-bot","Autn-WKOOP","Awario","AzureAI-SearchBot","barkrowler","bedrockbot","bigsur.ai","Bravebot","Brightbot","Brightbot 1.0","BuddyBot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT Agent","ChatGPT-User","Claude-Code","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","Cloudflare-AutoRAG","CloudVertexBot","Code","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Crawlspace","Datenbank Crawler","DeepSeekBot","Devin","Diffbot","dobot","DuckAssistBot","Echobot Bot","EchoboxBot","ExaBot","FacebookBot","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-Agent","Google-CloudVertexBot","Google-Extended","Google-Firebase","Google-Gemini-CLI","Google-NotebookLM","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","HawaiiBot","HenkBot","iAskBot","iaskspider","iaskspider/2.0","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","img2dataset","ISSCyberRiskCrawler","kagi-fetcher","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","LAIONDownloader","LCC","LinerBot","Linguee Bot","LinkupBot","Manus-User","meta-externalagent","meta-externalfetcher"]}
{"domain":"btloader.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"corriere.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":19,"stanza_tokens":["GPTBot","CCBot","anthropic-ai","AmazonAdBot","Google-CloudVertexBot","AppleBot","Google-Extended","Applebot-Extended","ClaudeBot","Amazonbot","Meta-ExternalAgent","PetalBot","Yandex","Bytespider","Scrapy","Diffbot-User","YouBot","Timpibot","*"]}
{"domain":"note.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","Googlebot","bingbot","Yeti","Twitterbot","Livelapbot","Megalodon","ia_archiver"]}
{"domain":"share-dns.com","fetch_ok":false,"usable":false,"reason":"other","status":409,"error":null,"attempts":["ENOTFOUND",409]}
{"domain":"amazon.com.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"nexusmods.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"ipify.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"heylink.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"investopedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"digitaloceanspaces.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"columbia.edu","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"xnxx.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kleinanzeigen.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","Adsbot-Google","Mediapartners-Google","zoomRank/2.0","Google-Display-Ads-Bot","OAI-SearchBot","GPTBot","ChatGPT-User","PerplexityBot","AmazonAdBot"]}
{"domain":"playrix.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"webempresa.eu","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"presage.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","ENOTFOUND"]}
{"domain":"nextlgsdp.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zenecn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cloudsink.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fbpigeon.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"4dex.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"onesignal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"typeform.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pbs.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":6,"stanza_tokens":["*","Baiduspider","GPTBot","CCBot","PerplexityBot","Bytespider"]}
{"domain":"smilewanted.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"segment.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Swiftbot"]}
{"domain":"agoda.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Googlebot","AdsBot-Google","google-hoteladsverifier","TurnitinBot","NPBot-1/2.0","NPBot","psbot","Mediapartners-Google"]}
{"domain":"binance.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["Baiduspider","Bingbot","DuckDuckBot","Slurp","Googlebot","Yandex","*"]}
{"domain":"xbox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"letsencrypt.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pixiv.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":13,"stanza_tokens":["*","GPTBot","Google-Extended","anthropic-ai","ClaudeBot","FacebookBot","Applebot-Extended","meta-externalagent","cohere-ai","CCBot","omgili","Diffbot","omgilibot"]}
{"domain":"faphouse.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"elmundo.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":117,"stanza_tokens":["*","008","AddThis.com","admantx","AhrefsBot","BDCbot","Bender","BIXOCRAWLER","bl.uk_lddc_bot","BLEXBot","BUbiNG","Cliqzbot","CNCDialer","crawler4j","CrystalSemanticsBot","CyberAlert","DigExt","discobot","discoverybot","dloader","dloader(NaverRobot)","DOC","dotbot","Download Ninja","DTS Agent","Exabot","Ezooms","FairShare","Fetch","Flamingo_SearchEngine","Genieo","Gigabot","grub-client","Heritrix","heritrix/3.3.0","HTTrack","ia_archiver","integromedb","IstellaBot","JikeSpider","Jyxobot","k2spider","Kimengi","Kimengi/nineconnections.com","larbin","LexxeBot/1.0","libwww","linko","Livelapbot","magpie-crawler","Maxthon","MetaURI","Microsoft.URL.Control","MJ12bot","Moreover","Moreoverbot","MSIECrawler","nabot","NaverBot","NerdByNature.Bot","netEstate NE Crawler","NetSeer crawler","Newscan","NextGenSearchBot","NPBot","nutch","Offline Explorer","omgilibot","Orthogaffe","PiplBot","Pixray-Seeker","proximic","psbot","QuerySeekerSpider","rogerbot","seokicks","SEOkicks-Robot","SiteBot","SiteBot/0.1","sitecheck.internetseer.com","SiteSnagger","Slurp","sogou","Sosospider","spbot","Spinn3r","Teleport","TeleportPro","trendictionbot","trovitBot","TurnitinBot","UbiCrawler","uMBot-LN","UnisterBot","UniversalFeedParser","WBSearchBot","WebCopier","WebReaper","WebStripper","WebZIP"]}
{"domain":"onet.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"netlify.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xhamster.desi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"kwai.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Baiduspider","360Spider","Sogouspider","Yisouspider","PetalBot","Bytespider"]}
{"domain":"privacy-mgmt.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tiktokpangle.us","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"samsungcloudsolution.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wyzecam.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":[429,429,429]}
{"domain":"tiktokrow-cdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"garmin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":12,"stanza_tokens":["*","gsa-garmin","Better Uptime Bot","DataForSeoBot","ecoresearchCrawler","EyeMonIT Uptime Bot","omgili","omgilibot","PerplexityBot","PiplBot","TurnitinBot","UptimeRobot"]}
{"domain":"ecosia.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","ia_archiver","Twitterbot"]}
{"domain":"kueezrtb.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"miwifi.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"readthedocs.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AhrefsBot"]}
{"domain":"rackspace.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nelreports.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"yandex.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","yandex","Googlebot","Twitterbot","Applebot"]}
{"domain":"xerox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","vspider"]}
{"domain":"meta.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["facebookexternalhit","meta-externalads","*","PetalBot","Scrapy","uptimerobot","viberbot","YaK","Yandex","Yeti"]}
{"domain":"youronlinechoices.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vk-analytics.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zhihu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["Googlebot","Googlebot-Image","bingbot","Baiduspider-news","Baiduspider","Baiduspider-render","Baiduspider-image","AdsBot-Google","Sogou web spider","Google-Extended","*"]}
{"domain":"quizlet.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ancestry.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","proximic","sogou spider","Google-Extended"]}
{"domain":"huffpost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":33,"stanza_tokens":["grapeshot","*","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","Diffbot","DuckAssistBot","FacebookBot","GPTBot","Google-Extended","Googlebot","magpie-crawler","meta-externalagent","Meta-ExternalFetcher","OAI-SearchBot","Perplexity-User","PerplexityBot","Timpibot","TurnitinBot","Ai2Bot-Dolma","Claude-SearchBot","Claude-User","omgilibot","TtoYouBot","WebGPTBot","Scope3/2.0 (scope3.com)","AmazonAdBot"]}
{"domain":"xhcdn.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"nextcloud.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rtbcdn.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nypost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":57,"stanza_tokens":["*","Bytespider","CCBot","claude-searchbot","claude-user","Claude-Web","ClaudeBot","FacebookBot","Google-Extended","MJ12bot","NewsNow","Perplexity-ai","PerplexityBot","PiplBot","anthropic-ai","cohere-ai","ia_archiver","magpie-crawler","news-please","omgili","omgilibot","meta-externalagent","meta-externalfetcher","A6-Indexer","AhrefsBot","Amazon-kendra","Amazon-qbusiness","amazonbot","amzn-searchbot","amzn-user","Applebot-Extended","archive.org_bot","AwarioRssBot","AwarioSmartBot","BLP_bbot","claritybot","DataForSeoBot","Diffbot","EasouSpider","ecoResearch","FriendlyCrawler","Genieo","ImagesiftBot","Meltwater","Meta-ExternalAgent","peer39_crawler","peer39_crawler/1.0","PetalBot","psbot","R6_CommentReader","Scrapy","Seekr","Spinn3r","TurnitinBot","WebVac","WebZip","FontReport"]}
{"domain":"snapkit.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ox.ac.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"kwcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"docomo.ne.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"immedia-semi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"amemv.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"netgear.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"infobae.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dnse0.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bidmachine.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"flipkart.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"heytapmobi.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"tailscale.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","atlassian-bot"]}
{"domain":"fwmrm.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bol.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","Baiduspider","omgili","Slurp","rogerbot","YandexBot","PetalBot"]}
{"domain":"kslawin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Baiduspider","360Spider","Sogouspider","Yisouspider","PetalBot","Bytespider"]}
{"domain":"playfabapi.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"ubi.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"mybluehost.me","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","attempts":["ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"bing.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"google.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"as.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["PetalBot","bedrock-agentcore-browser","Amazon-Bot","Amazonbot","AmazonProductDiscoveryBot","Amzn-SearchBot","Amzn-User","Arquivo-web-crawler","bnf.fr_bot","Bytespider","Claude-Extension","ClaudeBot","Claude-User","Claude-SearchBot","Comet","CloudFlare-AlwaysOnline","Cloudflare-Healthchecks","CloudflareBrowserRendering","Cloudflare-Radar","CCBot","DeepSeekBot","DuckAssistBot","archive.org_bot","ia_archiver","MistralAI-User","Moreover","Adbeat","Argus","Baiduspider","Baiduspider-image","Bitdefender","Blogtrottr","BloombergBot","BoardReader","checkmy.ws","Clickagy","CookieBot","cXensebot","DataObserver","DomainStatsBot","Embedly","Feedspotbot","FlipboardProxy","Getintent","Google-Apps-Script","GoogleDrive","Grapevine6","Gumgum","HaloBot","HubSpot Crawler","IbouBot","Inout","LinerBot","MJ12bot","Manus","Meltwater","Metajob","Muckrack","NetSeer","Netvibes","Onyphe","OpenClaw","Opoint","OutbrainBot","PlumAnalytics","ProSitemaps","PerplexityBot","Perplexity-User","proximic","PubMatic","Quantcastbot","SearchConcepts","Semantic-Visions","SentiBot","Sindup","Sirdata","SmartologyBot","Superfeedr","surlybot","Synaptive","TinEye","Trendiction","TurnitinBot","Ubermetrics","Vebidoo","Webcorp","Weborama","WebPageTest","YandexAccessibilityBot","YandexDirect","YandexDirectDyn","YaDirectFetcher","YandexFavicons","YandexMetrika","YandexMobileBot","YandexNews","YandexRCA","YandexRenderResourcesBot","YandexUserproxy","*"]}
{"domain":"xing.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["CrazyWebCrawler-Spider","*","pimeyes.com crawler","GPTBot","GPTUser","ClaudeBot","Claude-User","PerplexityBot","Perplexity-User"]}
{"domain":"sofascore.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"rambler.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Yandex","Googlebot","GPTBot","Shodan","Anews/1.0","*"]}
{"domain":"cdnhwc2.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","CERT_HAS_EXPIRED"]}
{"domain":"yximgs.com","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":["ENOTFOUND",400]}
{"domain":"volcgslb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"att.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"lg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["ChatGPT-User","OAI-SearchBot","PerplexityBot","ClaudeBot","GoogleOther","GPTBot","Google-Extended","BingPreview","Googlebot","Googlebot-image","*","Baiduspider","Yandex","Mediapartners-Google","AdsBot-Google","AdsBot-Google-Mobile"]}
{"domain":"i18n-pglstatp.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"redfin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["trovitBot","Exabot","sogou spider","Sosospider","peer39_crawler/1.0","AdsBot-Google","AdsBot-Google-Mobile","google-amphtml","AmazonAdBot","FireCrawlAgent","bingbot","*"]}
{"domain":"pvp.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gartner.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"app-analytics-services-att.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"letterboxd.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":25,"stanza_tokens":["AI2Bot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Diffbot","cohere-training-data-crawler","Cotoyogi","Datenbank Crawler","FacebookBot","Factset_spyderbot","Google-Extended","GPTBot","ICC-Crawler","Kangaroo Bot","Meta-ExternalAgent","netEstate Imprint Crawler","omgili","PanguBot","PetalBot","SemrushBot-OCOB","Timpibot","Webzio-Extended","Algolia Crawler","*"]}
{"domain":"oecd.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Twitterbot","Googlebot","Spacecat/1.0"]}
{"domain":"goal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"coursera.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","CCBot","GPTBot","ClaudeBot","MistralAI-User","Google-Extended","PerplexityBot","LinkedInBot","Twitterbot","meta-externalagent","OAI-SearchBot","ChatGPT-User"]}
{"domain":"irs.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"us.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"moneycontrol.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"n-tv.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":4,"stanza_tokens":["*","GPTBot","Google-Extended","CCBot"]}
{"domain":"marca.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":115,"stanza_tokens":["*","008","AddThis.com","admantx","BDCbot","Bender","BIXOCRAWLER","bl.uk_lddc_bot","BLEXBot","BUbiNG","Cliqzbot","CNCDialer","crawler4j","CrystalSemanticsBot","CyberAlert","DigExt","discobot","discoverybot","dloader","dloader(NaverRobot)","DOC","dotbot","Download Ninja","DTS Agent","Exabot","Ezooms","FairShare","Fetch","Flamingo_SearchEngine","Genieo","Gigabot","grub-client","Heritrix","heritrix/3.3.0","HTTrack","ia_archiver","integromedb","IstellaBot","JikeSpider","Jyxobot","k2spider","Kimengi","Kimengi/nineconnections.com","larbin","LexxeBot/1.0","libwww","linko","Livelapbot","magpie-crawler","Maxthon","MetaURI","Microsoft.URL.Control","MJ12bot","Moreover","Moreoverbot","MSIECrawler","nabot","NaverBot","NerdByNature.Bot","netEstate NE Crawler","NetSeer crawler","Newscan","NextGenSearchBot","NPBot","nutch","Offline Explorer","omgilibot","Orthogaffe","PiplBot","Pixray-Seeker","proximic","psbot","QuerySeekerSpider","rogerbot","seokicks","SEOkicks-Robot","SiteBot","SiteBot/0.1","sitecheck.internetseer.com","SiteSnagger","Slurp","sogou","Sosospider","spbot","Spinn3r","Teleport","TeleportPro","trendictionbot","trovitBot","TurnitinBot","UbiCrawler","uMBot-LN","UnisterBot","UniversalFeedParser","WBSearchBot","WebCopier","WebReaper","WebStripper","WebZIP","WeSEE:Search"]}
{"domain":"statcounter.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"alphonso.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Googlebot-Image","ia_archiver","SemrushBot","MJ12bot","AhrefsBot"]}
{"domain":"johndeerecloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"epicgames.dev","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,404]}
{"domain":"footprintdns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"clever.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dribbble.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","PetalBot"]}
{"domain":"fortune.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","Googlebot","LinkedInBot","CCBot","Google-Extended","ia_archiver","omgili","omgilibot"]}
{"domain":"adblockplus.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","008","Linguee","mxbot","ip-web-crawler.com","Screaming Frog SEO Spider"]}
{"domain":"lwsdns.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"trendyol.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Googlebot","Googlebot-Image","AdsBot-Google-Mobile","AdsBot-Google","ia_archiver","archive.org_bot","ia_archiver-web.archive.org","AhrefsBot"]}
{"domain":"plos.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ovscdns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"apa.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["atlassian-bot","*","sogou spider","Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm","Sogou+web+spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm","Sougou+web+spider"]}
{"domain":"ilovepdf.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tabelog.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":20,"stanza_tokens":["*","Baiduspider","BaiduMobaider","BaiduImagespider","bingbot","Yeti","psbot","BecomeBot","Teoma","Ask Jeeves","ListRan","GPTBot","grapeshot","YouBot","Leikibot","Sogou","Google-Extended","Mediapartners-Google","Googlebot","AmazonAdBot"]}
{"domain":"singular.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","anthropic-ai","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Amazonbot","*"]}
{"domain":"crazygames.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","SemrushBot"]}
{"domain":"yale.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hilton.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"wise.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":83,"stanza_tokens":["*","Rogerbot","Exabot","MJ12bot","Dotbot","Gigabot","AhrefsBot","BlackWidow","ChinaClaw","Custo","DISCo","Download\\ Demon","eCatch","EirGrabber","EmailSiphon","EmailWolf","Express\\ WebPictures","ExtractorPro","EyeNetIE","FlashGet","GetRight","GetWeb!","Go!Zilla","Go-Ahead-Got-It","GrabNet","Grafula","HMView","HTTrack","Image\\ Stripper","Image\\ Sucker","Indy\\ Library","InterGET","Internet\\ Ninja","JetCar","JOC\\ Web\\ Spider","larbin","LeechFTP","Mass\\ Downloader","MIDown\\ tool","Mister\\ PiX","Navroad","NearSite","NetAnts","NetSpider","Net\\ Vampire","NetZIP","Octopus","Offline\\ Explorer","Offline\\ Navigator","PageGrabber","Papa\\ Foto","pavuk","pcBrowser","RealDownload","ReGet","SiteSnagger","SemrushBot","SmartDownload","SuperBot","SuperHTTP","Surfbot","tAkeOut","Teleport\\ Pro","VoidEYE","Web\\ Image\\ Collector","Web\\ Sucker","WebAuto","WebCopier","WebFetch","WebGo\\ IS","WebLeacher","WebReaper","WebSauger","Website\\ eXtractor","Website\\ Quester","WebStripper","WebWhacker","WebZIP","Wget","Widow","WWWOFFLE","Xaldon\\ WebSpider","Zeus"]}
{"domain":"mercadolibre.com.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User"],"open":[],"path_restricted":["Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":13,"stanza_tokens":["Amazonbot","PerplexityBot","Perplexity-User","ClaudeBot","Claude-User","GPTBot","ChatGPT-User","FacebookExternalHit","FacebookBot","Twitterbot","LinkedInBot","Bingbot","*"]}
{"domain":"ameba.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["Googlebot"]}
{"domain":"yandex.kz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","yandex","Twitterbot"]}
{"domain":"ouest-france.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":181,"stanza_tokens":["Mediapartners-Google","Googlebot","Googlebot-Image","Googlebot-News","Googlebot-Video","GoogleOther","Adsbot-Google","Storebot-Google","AdsBot-Google-Mobile","APIs-Google","Twitterbot","Applebot","Ouestfrancebot","Taboolabot","Proximic","upday","Bingbot","exabot","Slurp","ia_archiver","grapeshot","facebookexternalhit","Qwantbot","cXensebot","AmazonAdBot","IbouBot","ChatGPT-User","Claude-User","Claude-SearchBot","MistralAI-User","PerplexityBot","Perplexity-User","DuckAssistBot","Gemini-DeepResearch","Applebot-Extended","ClaudeBot","Google-Extended","Google-NotebookLM","FacebookBot","Amazonbot","Bytespider","OAI-SearchBot","Meta-ExternalAgent","Meta-ExternalFetcher","Meta-WebIndexer","5emeRue","5erue","adequat","adequat-systems","AgentCore-Browser","Ai2Bot-Dolma","AI2Bot/v2.0","Aihitdata.com","AmazonBot","AmiSoftware","amazon-kendra","anthropic-ai","Argus","Asknread","asknread.com","Augure","auramundi","bedrockbot/3.0","Bloodhound","CCBot","ChatGPT-Agent","ChatGLM-Spider","Channel3Bot","Cision","Claude-Web","coexel","Cohere-ai","ConveraCrawler","Corporama","CotoyogiBot","cydralspider","DeepSeekBot","Diffbot","Digimind","DownloadNinja","downloadexpress","EDD","ellisphere","eureka","Europresse","Explore","Factiva","Fasterfox","Fetch","gammaSpider","GoogleAgent-Shopping","Google-CloudVertextBot","GPTBot","grub-client","HTTrack","IAskBot","ia_archiver-web.archive.org","ICC-Crawler","ImageSiftBot","img2dataset"]}
{"domain":"bfmtv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":84,"stanza_tokens":["*","Googlebot-News","ChatGPT-User","OAI-SearchBot","Claude-User","Claude-Web","Claude-SearchBot","Perplexity-User","Google-Extended","Gemini-Deep-Research","MistralAI-User","GPTBot","Operator","ClaudeBot","anthropic-ai","GoogleOther","Google-CloudVertexBot","GoogleOther-Image","GoogleOther-Video","Applebot-Extended","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","Amazonbot","bedrockbot","NovaAct","PerplexityBot","cohere-ai","cohere-training-data-crawler","Bytespider","TikTokSpider","CCBot","Diffbot","omgili","omgilibot","Webzio-Extended","YouBot","PetalBot","PanguBot","SemrushBot-OCOB","SemrushBot-SWA","ImagesiftBot","img2dataset","FirecrawlAgent","ShapBot","Thinkbot","VelenPublicWebCrawler","Andibot","PhindBot","YandexAdditional","YandexAdditionalBot","FriendlyCrawler","Crawlspace","ICC-Crawler","SBIntuitionsBot","Panscient","panscient.com","Timpibot","AddSearchBot","aiHitBot","iaskspider","LinerBot","CloudVertexBot","Google-Firebase","Terrecotta","TerraCotta","Kangaroo Bot","meta-webindexer","Brightbot","Brightbot 1.0","SearchmetricsBot","deepcrawl","rogerbot","MJ12bot","OrangeBot","OrangeBot-Collector","Slurp","Adixbot","YandexBot","AhrefsBot","Sistrix","SemrushBot","Screaming Frog SEO Spider"]}
{"domain":"leiniao.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"hugedomains.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","MJ12bot","dotbot"]}
{"domain":"nexthink.cloud","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"yastatic.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mythad.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"warnerbros.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"vercel-dns-3.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"webhostbox.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"usercentrics.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"incapdns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cookpad.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"olx.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hotstar.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"inkuai.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"lge.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["ChatGPT-User","OAI-SearchBot","PerplexityBot","ClaudeBot","GoogleOther","GPTBot","Google-Extended","BingPreview","Googlebot","Googlebot-image","*","Baiduspider","Yandex","Mediapartners-Google","AdsBot-Google","AdsBot-Google-Mobile"]}
{"domain":"gsmarena.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["OAI-SearchBot","ChatGPT-User"],"open":[],"path_restricted":["GPTBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":24,"stanza_tokens":["*","Mediapartners-Google","AppEngine-Google","Yandex","AhrefsBot","bingbot","slurp","Seznambot","DataForSeoBot","dotbot","grapeshot","ChatGPT-User","ImagesiftBot","OAI-SearchBot","MJ12bot","Amazonbot","ClaudeBot","meta-externalagent","CCBot","PerplexityBot","Diffbot","anthropic-ai","YouBot","FeedBurner"]}
{"domain":"sectigo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tnaflix.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"idnes.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":17,"stanza_tokens":["*","MachineLearning","GPTBot","Google-Extended","PerplexityBot","Amazonbot","ClaudeBot","Omgilibot","FacebookBot","Applebot","anthropic-ai","Bytespider","Claude-Web","Diffbot","ImagesiftBot","Omgili","YouBot"]}
{"domain":"economist.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":18,"stanza_tokens":["GPTBot","Google-Extended","ChatGPT-User","CCBot","PiplBot","anthropic-ai","Claude-Web","TurnitinBot","PetalBot","MoodleBot","magpie-crawler","ia_archiver","Applebot-Extended","PerplexityBot","Perplexity-User","Bytespider","Mozilla/5.0 (compatible; parse.ly scraper/0.16; +http://parsely.com)","*"]}
{"domain":"tawk.to","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"aparat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"skysports.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":6,"stanza_tokens":["*","AhrefsBot","CCBot","GPTBot","Yandexbot","AmazonAdBot"]}
{"domain":"jstor.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":6,"stanza_tokens":["Googlebot","*","ClaudeBot","CCBot","GPTBot","Google-Extended"]}
{"domain":"xhamster2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"pandora.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["PiplBot","bingbot","deepcrawl","Googlebot","*"]}
{"domain":"frontiersin.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"samsungcloud.tv","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"magnific.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Mediapartners-Google","Adsbot-Google","Twitterbot"]}
{"domain":"nic.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ETIMEDOUT"]}
{"domain":"espncricinfo.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"lanacion.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["TestBot","AmazonAdBot","*"]}
{"domain":"onedrive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"byteoversea.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"upenn.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"finn.no","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Baiduspider","Twitterbot","IndeedBot","Jobblebot","neuvoo","GPTBot","OAI-SearchBot","ChatGPT-User"]}
{"domain":"olx.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"internetwarriors.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"hltv.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sahibinden.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"psychologytoday.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["sosospider","yandex bot","larbin","BizwikiBot","ScoutJet","Riddlerbot","SemrushBot","ShopWiki","ia_archiver","*"]}
{"domain":"clarin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"archiveofourown.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":6,"stanza_tokens":["*","Googlebot","CCBot","GPTBot","ChatGPT-User","Slurp"]}
{"domain":"rbc.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","GoogleBot","Yandex"]}
{"domain":"ultimate-guitar.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"meethue.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Baiduspider","360Spider","Sogou web spider","ISS_CRAWLER_V1"]}
{"domain":"sentinelone.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pendo.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xhamster19.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"phncdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"skroutz.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":23,"stanza_tokens":["*","Yandex","YandexBot","YandexMobileBot","YandexDirect","APIs-Google","AdsBot-Google-Mobile","AdsBot-Google","Mediapartners-Google","Amazonbot","PetalBot","Bytespider","anthropic-ai","CCBot","Baiduspider-render","Baiduspider","OAI-SearchBot","GPTBot","ChatGPT-User/2.0","Google-Extended","PerplexityBot","ClaudeBot","Claude-SearchBot"]}
{"domain":"bund.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sapo.pt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":49,"stanza_tokens":["*","AddSearchBot","AI2Bot","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","bigsur.ai","Bytespider","CCBot","ChatGPT Agent","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Datenbank Crawler","Devin","Diffbot","DuckAssistBot","FacebookBot","Gemini-Deep-Research","Google-Extended","GoogleAgent-Mariner","GoogleOther","Googlebot","Bingbot","GPTBot","ICC-Crawler","Kangaroo Bot","LinerBot","meta-externalagent","MistralAI-User","netEstate Imprint Crawler","NovaAct","OAI-SearchBot","omgili","PanguBot","Perplexity-User","PerplexityBot","PetalBot","QualifiedBot","Timpibot","Webzio-Extended","YouBot"]}
{"domain":"cspserver.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"hostinger.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"riotgames.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"flashscore.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"namu.wiki","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"dafont.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"huffingtonpost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":33,"stanza_tokens":["grapeshot","*","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","Diffbot","DuckAssistBot","FacebookBot","GPTBot","Google-Extended","Googlebot","magpie-crawler","meta-externalagent","Meta-ExternalFetcher","OAI-SearchBot","Perplexity-User","PerplexityBot","Timpibot","TurnitinBot","Ai2Bot-Dolma","Claude-SearchBot","Claude-User","omgilibot","TtoYouBot","WebGPTBot","Scope3/2.0 (scope3.com)","AmazonAdBot"]}
{"domain":"namecheaphosting.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"susercontent.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"superhosting.bg","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kicker.de","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"fc2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"biomedcentral.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[404,200]}
{"domain":"vivo.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vg.no","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":39,"stanza_tokens":["*","OAI-SearchBot","Googlebot-News","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","Claude-Web","ClaudeBot","Claude-SearchBot","cohere-ai","cohere-training-data-crawler","Crawlspace","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","Kangaroo Bot","Meta-ExternalAgent","omgili","omgilibot","PanguBot","PerplexityBot","PetalBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot"]}
{"domain":"eu-1-id5-sync.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"dmm.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aftonbladet.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":38,"stanza_tokens":["*","OAI-SearchBot","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","Claude-Web","ClaudeBot","Claude-SearchBot","cohere-ai","cohere-training-data-crawler","Crawlspace","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","Kangaroo Bot","Meta-ExternalAgent","omgili","omgilibot","PanguBot","PerplexityBot","PetalBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot"]}
{"domain":"gismeteo.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"akamaitechnologies.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"medallia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"company-target.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","attempts":["ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","ENOTFOUND"]}
{"domain":"tubi.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"xhamster1.desi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"fb.me","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"afafb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"triplinkintl.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"gofundme.com","fetch_ok":false,"usable":false,"reason":"other","status":502,"error":null,"attempts":[502,502,502]}
{"domain":"indiamart.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":57,"stanza_tokens":["*","AdIdxbot","Facebookbot","Facebot","ClaudeBot","CriteoBot/0.1","alexabot","ia_archiver","alexa site audit","Bytespider","omgili","Dataprovider.com","dcrawl","Nutch","HTTrack","HTTrack 3.0","MetaInspector","Offline Explorer","008","Slurp","SeznamBot","SearchmetricsBot","Feedonomics","EtaoSpider","BLEXBot","ImagesiftBot","trendictionbot","EyeMonIT Uptime Bot","AhrefsSiteAudit","Mail.RU_Bot","AhrefsBot","Baiduspider","SputnikBot","MJ12bot","FeedBurner","Exabot","proximic","Scrapy","SemrushBot","coccocbot","IAS Crawler","dotbot","ltx71","Sogou web spider","Pingdom.com_bot","GetIntent Crawler","expo9","PetalBot","Yandex","bingbot","Mediapartners-Google","Adsbot-Google","OAI-SearchBot","GPTBot","ChatGPT-User","Google-Extended","meta-externalagent"]}
{"domain":"dnse2.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"jw.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"elconfidencial.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":61,"stanza_tokens":["*","FairShare","MetaURI","nabot","NaverBot","Kimengi/nineconnections.com","Kimengi","DigExt","DTS Agent","dloader","dloader(NaverRobot)","magpie-crawler","MJ12bot","AhrefsBot","Exabot","QuerySeekerSpider","UniversalFeedParser","Flamingo_SearchEngine","Livelapbot","Wotbot","Wotbox","BIXOCRAWLER","proximic","Genieo","Heritrix","rogerbot","spbot","trendictionbot","uMBot-LN","SEOkicks-Robot","nutch","crawler4j","dotbot","Moreover","omgilibot","Cliqzbot","BUbiNG","PiplBot","Gigabot","GPTBot","ChatGPT-User","CCBot","anthropic-ai","ClaudeBot","PerplexityBot","Google-Extended","Applebot-Extended","Meta-ExternalAgent","Meta-ExternalFetcher","Baiduspider","AI-SearchBot","cohere-ai","OAI-SearchBot","Bytespider","Amazonbot","Diffbot","YouBot","PetalBot","Timpibot","DataForSeoBot","omgili"]}
{"domain":"dcinside.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":16,"stanza_tokens":["GPTBot","ClaudeBot","anthropic-ai","Claude-Web","Google-Extended","Applebot-Extended","CCBot","Bytespider","Amazonbot","Meta-ExternalAgent","cohere-ai","PerplexityBot","Baiduspider","Baiduspider-video","Baiduspider-image","*"]}
{"domain":"vinted.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":7,"stanza_tokens":["*","grapeshot","GPTBot","ClaudeBot","Google-Extended","Amazonbot","CCBot"]}
{"domain":"he.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"rule34.xxx","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"lequipe.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":103,"stanza_tokens":["*","AI2Bot","Adequat","Alexibot","AlvinetSpider","AmiSoftware","anthropic-ai","Applebot-Extended","Argus","AudigentAdBot","Augure","BeOpBot","BizInformation","Black Hole","BunnySlippers","Bytespider","CCBot","Cision","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","Coexel","cohere-ai","cohere-training-data-crawler","Corporama","DataForSeoBot","Diffbot","Digimind","dotbot","Download Ninja","DuckAssistBot","Ellisphere","EmailCollector","EmailSiphon","EmailWolf","FacebookBot","Factiva","Fetch","GPTBot","Gemini-Deep-Research","Google-Extended","HTTrack","Jetbot","Kangaroo Bot","kbcrawl","linkfluence","lwp-trivial","Manageo","Meltwater","Mention","Meta-ExternalAgent","Meta-ExternalFetcher","MistralAI-User","moget","Moreover","Naverbot","NetAnts","NetAttache","NetMechanic","NiceRsPRO","Offline Explorer","Onclusive","PanguBot","Perplexity-User","PerplexityBot","Proxem","psbot","Qwam content intelligence","Raven","scoop.it","Score3","SemrushBot","Sindup","SiteSucker","Sogou web spider","Spotter","SuperBot","Synthesio","Talkwalker","Teleport","TeleportPro","Trendeo","Trendybuzz","True_Robot","UptimeRobot","UrlPouls","VCI","Vecteurplus","VerticalSearch","WebBandit","WebCopier","WebReaper","WebSauger","WebStripper","WebZIP","Webzio-Extended","Wget","WinHTTrack","Winello"]}
{"domain":"xhaccess.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"psu.edu","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"hotpepper.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","bingbot","Slurp"]}
{"domain":"vnexpress.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":54,"stanza_tokens":["*","AliyunSecBot","AmazonBot","anthropic-ai","Applebot-Extended","archive.org_bot","AudigentAdBot","AwarioRssBot","AwarioSmartBot","BLEXBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","DuckAssistBot","EchoboxBot","FriendlyCrawler","Baiduspider","Bingbot","Slurp","GPTBot","ImagesiftBot","Jetslide","magpie-crawler","MyCentralAIScraperBot","NewsNow","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Perplexity-User","Poseidon Research Crawler","quillbot.com","Quora-Bot","Scrapy","SeekrBot","SeznamHomepageCrawler","TaraGroup Intelligent Bot","Timpibot","TurnitinBot","ViennaTinyBot","YouBot","Google-Extended","Google-CloudVertexBot","facebookexternalhit"]}
{"domain":"spankbang.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"nu.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":39,"stanza_tokens":["*","Googlebot-News","AhrefsBot","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude","Claude-Web","ClaudeBot","coccocbot-web","cohere-ai","DeepSeekBot","Diffbot","Exabot","Google-Extended","GPTBot","Meta-ExternalFetcher","MistralAI-User","MistralAI-User/1.0","MJ12bot","OAI-SearchBot","Perplexity-User","PerplexityBot","PetalBot","proximic","SemrushBot","SemrushBot-OCOB","SemrushBot-SWA","Timpibot","Yandex","YandexAdditional","YandexAdditionalBot","YandexBot","YandexImages","YandexRenderBot","YouBot"]}
{"domain":"alipay.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[404,200]}
{"domain":"itmedia.co.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"tbs.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ttlivecdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"abc.net.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":31,"stanza_tokens":["*","Googlebot","Googlebot-Image","MSNBot","Slurp","FlipboardProxy","iSec_Bot","TurnitinBot","ICC-Crawler","trendkite-akashic-crawler","TinEye-bot","R6_CommentReader","BLEXBot","Nutch","ChatGPT-User","ClaudeBot","Claude-Web","Timpibot","omgili","cohere-ai","anthropic-ai","Bytespider","CCBot","Diffbot","Meta-ExternalAgent","Google-Extended","GPTBot","PerplexityBot","Omgilibot","Webzio-Extended","YouBot"]}
{"domain":"ad.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":30,"stanza_tokens":["*","Twitterbot","GPTBot","ChatGPT-User","CCBot","anthropic-ai","Amazonbot","Applebot-Extended","Bytespider","ClaudeBot","Claude-Web","cohere-ai","Diffbot","Google-Extended","Meta-ExternalFetcher","PerplexityBot","YouBot","DeepSeekBot","MistralAI-User","MistralAI-User/1.0","PetalBot","Timpibot","YandexAdditional","YandexAdditionalBot","YandexBot","YandexRenderBot","Perplexity-User","coccocbot-web","OAI-SearchBot","Claude"]}
{"domain":"service-now.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ECONNRESET"]}
{"domain":"cnnbrasil.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"janitorai.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"leboncoin.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["Googlebot","Googlebot-Mobile","Yahoo! Slurp","Slurp","slurp","msnbot","bingbot","ia_archiver","AdsBot-Google","AdsBot-Google-Mobile","Mediapartners-Google","proximic","Google-Extended","OAI-SearchBot","ChatGPT-User","facebookexternalhit"]}
{"domain":"fb-t-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"lichess.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bitdefender.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"furaffinity.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":4,"stanza_tokens":["GPTBot","ChatGPT-User","CCBot","*"]}
{"domain":"google.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"google.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"boyfriendtv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AhrefsBot","PetalBot"]}
{"domain":"fmkorea.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":57,"stanza_tokens":["AhrefsBot","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot-Extended","BLEXBot","BoardReader","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","DeepSeek","DeepSeekBot","Diffbot","dotbot","FacebookBot","Google-Extended","GPTBot","ia_archiver","ICCrawler","ImagesiftBot","Linguee Bot","Meta-ExternalAgent","MJ12bot","Omgili","Omgilibot","PerplexityBot","PetalBot","SemrushBot","Sogou inst spider","Sogou web spider","The Knowledge AI","trendictionbot","TurnitinBot","Tweetmemebot","YouBot","webzio","Webzio-Extended","AmazonAdBot","facebookexternalhit","ChatGPT-User","Claude-User","Claude-SearchBot","OAI-SearchBot","Perplexity-User","Mediapartners-Google","Twitterbot","Googlebot","Yeti","Daum","ZumBot","bingbot","Applebot","DuckDuckBot","*"]}
{"domain":"safety.google","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pornpics.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ECONNRESET"]}
{"domain":"tradplusad.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"postgresql.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kakao.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["GoogleOther","*"]}
{"domain":"xhopen.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"rightmove.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":11,"stanza_tokens":["*","AmazonAdBot","GPTBot","MJ12bot","Mediapartners-Google","SpriftCrawler","TrovitBot","cbBot","oodlebot","spider","CCbot"]}
{"domain":"ddns.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UND_ERR_CONNECT_TIMEOUT","attempts":["UND_ERR_CONNECT_TIMEOUT","TimeoutError"]}
{"domain":"bmj.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"donga.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"websitewelcome.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"amazon.co.za","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"trello.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"admin.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pccc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ynet.co.il","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bilibili.tv","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"meesho.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"icanhazip.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"tesla.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"wordwall.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"livescore.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","barkrowler","SiteAuditBot","PetalBot","Bytespider","ChatGPT-User"]}
{"domain":"acesso.gov.br","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"apigee.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"opera-api.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"txnhh.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hm.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bestbuy.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"adaether.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"split.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"remove.bg","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot","Googlebot-image"]}
{"domain":"gazzetta.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":15,"stanza_tokens":["psbot","NPBot-1/2.0","NPBot","Baiduspider","Grok","Google-CloudVertexBot","AmazonAdBot","GPTBot","CCBot","Google-Extended","Applebot-Extended","ClaudeBot","Amazonbot","Meta-ExternalAgent","*"]}
{"domain":"stbid.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zdnscloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"fetlife.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vscode-cdn.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"nic.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"b7510.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"marktplaats.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"porkbun.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xiaohongshu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["Googlebot","Baiduspider","bingbot","Sogou web spider","Sogou wap spider","YisouSpider","BaiduSpider-ads","*"]}
{"domain":"blackberry.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Uptimebot"]}
{"domain":"businesswire.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"futbin.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"nsone.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"xhamster45.desi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"tabor.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","*","MJ12bot"]}
{"domain":"xhname.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"idealista.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"litnet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["YandexBot","*"]}
{"domain":"poste.it","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"caixa.gov.br","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"msauth.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"costco.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"tapad.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"cricbuzz.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"argos.co.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"dw.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":52,"stanza_tokens":["*","Googlebot","Bingbot","AhrefsBot","Ai2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","dotbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","ia_archiver","ImagesiftBot","img2dataset","magpie-crawler","Meta-ExternalAgent","Meta-ExternalFetcher","NewsNow","MJ12bot","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","PetalBot","Scrapy","SemrushBot","SirdataBot","Timpibot","TurnitinBot","VelenPublicWebCrawler","Webzio-Extended","Yeti","YouBot","Twitterbot","facebookexternalhit"]}
{"domain":"mydramalist.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"merkur.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":248,"stanza_tokens":["*","Amazonbot","AlibabaBot","Ai2Bot-Dolma","Applebot-Extended","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","Claude-Web","CloudVertexBot","cohere-training-data-crawler","Cotoyogi","Datenbank Crawler","Diffbot","FacebookBot","Google-Extended","GoogleOther","GPTBot","ICC-Crawler","imageSpider","Kangaroo Bot","laion-huggingface-processor","LCC","meta-externalagent","netEstate Imprint Crawler","omgili","PanguBot","PerplexityBot","SBIntuitionsBot","Spider","Timpibot","VelenPublicWebCrawler","webzio-extended","YouBot","008","Dataprovider.com","dcrawl","HelloworkJobPostingBot","HTTrack","HTTrack 3.0","KrawlerBot","MetaInspector","MetaJobBot","newspaper","Nutch","Offline Explorer","OpenindexSpider","Potions","Scrapy","ServerHunterSpider","StatsDroneBot","CloudflareBrowserRenderingCrawler","aa","AhrefsBot","Attracta","Barkrowler","BrightEdge Crawler","BLEXBot","CaliberBot","Caliperbot","ClarityBot","Clearscopebot","Cloudtrellis","cludo.com","Cocolyzebot","cognitiveSEO Crawler","contentking","Convermax","Cxense","DataForSeoBot","DataForSEO Bot","DomainStatsBot","DotBot","Dragonbot","Huckabot","Huckabuy Bot","hypestat","iaskspider","img2dataset","James BOT","LinkCheck by Siteimprove","LinkChecker Bot","linkchecker.pro","linkdexbot","LinksIndexerBot","MarketGoo","MBCrawler","MegaIndex.ru","MJ12bot","Moz dotbot","Moz rogerbot","Nitro-","NitroBot","online-webceo-bot","Prerender","Pro Sitemaps","Readable","RevvimGort","rogerbot-crawler"]}
{"domain":"zalando.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Pinterestbot","*"]}
{"domain":"1c.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","SemrushBot","SemrushBot-SA","Riddler","BLEXBot","serpstatbot","bingbot","Bytespider","PetalBot"]}
{"domain":"vietnam.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"interia.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","grapeshot"]}
{"domain":"canada.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"singtelchina.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dlsite.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["dotbot","Eyeotabot","*"]}
{"domain":"ucla.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wiktionary.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["MJ12bot","Mediapartners-Google*","IsraBot","Orthogaffe","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","fast","wget","grub-client","k2spider","NPBot","WebReaper","SemrushBot","*"]}
{"domain":"stackexchange.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":418,"error":null,"attempts":[418,418]}
{"domain":"labs.google","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"googleapis.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"pimpbunny.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":37,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","OAI-SearchBot","ChatGPT-User","anthropic-ai","Meta-ExternalAgent","Meta-ExternalFetcher","FacebookBot","PerplexityBot","cohere-ai","cohere-training-data-crawler","AI2Bot","Diffbot","DuckAssistBot","Webzio-Extended","Omgilibot","ImagesiftBot","Kangaroo Bot","PanguBot","Timpibot","YouBo","Googlebot","Bingbot","DuckDuckBot","Yandex","Baiduspider","Bravebot","AhrefsBot","AhrefsSiteAudit"]}
{"domain":"parklogic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"inven.co.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","Wget","*"]}
{"domain":"usnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":56,"stanza_tokens":["*","Googlebot-News","Applebot","Bingbot","Amazonbot","amazon-kendra","bedrockbot","Anthropic-ai","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","Cohere-ai","DataForSeoBot","FacebookBot","meta-externalagent","meta-externalfetcher","Google-Extended","Gemini-Deep-Research","ImagesiftBot","Magpie-crawler","Omgili","Omgilibot","PerplexityBot","Perplexity-User","YouBot","huggingface","NewsNow","news-please","Diffbot","Scrapy","TurnitinBot","Applebot-Extended","AI2Bot","MistralAI-User","ChatGLM-Spider","Cotoyogi","DeepSeekBot","EchoboxBot","iaskspider","ICC-Crawler","LinerBot","PanguBot","ProRataInc","QualifiedBot","SeekrBot","VelenPublicWebCrawler","WARDBot","WPBot","WRTNBot","ZanistaBot"]}
{"domain":"livedoor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"spbycdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kernel.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"hln.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":30,"stanza_tokens":["*","Twitterbot","GPTBot","ChatGPT-User","CCBot","anthropic-ai","Amazonbot","Applebot-Extended","Bytespider","ClaudeBot","Claude-Web","cohere-ai","Diffbot","Google-Extended","Meta-ExternalFetcher","PerplexityBot","YouBot","DeepSeekBot","MistralAI-User","MistralAI-User/1.0","PetalBot","Timpibot","YandexAdditional","YandexAdditionalBot","YandexBot","YandexRenderBot","Perplexity-User","coccocbot-web","OAI-SearchBot","Claude"]}
{"domain":"lc.chat","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"myntra.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"excite.co.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"jusbrasil.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"notion.so","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","BLEXBot","AhrefsBot","Amazonbot","SemrushBot","dotbot"]}
{"domain":"utorrent.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","GPTBot","ClaudeBot","PerplexityBot","Google-Extended"]}
{"domain":"noodlemagazine.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"classlink.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"everesttech.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"buzzfeed.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":35,"stanza_tokens":["msnbot","*","Amazonbot","anthropic-ai","Applebot-Extended","CCBot","ClaudeBot","Claude-Web","cohere-ai","Diffbot","discobot","dotbot","FacebookBot","google-extended","Google-Extended","Googlebot","GPTBot","PerplexityBot","TurnitinBot","magpie-crawler","meta-externalagent","Meta-ExternalFetcher","Bytespider","DuckAssistBot","Perplexity-User","Timpibot","Ai2Bot-Dolma","omgilibot","TtoYouBot","WebGPTBot","Slurp","Verity/1.1","Scope3/2.0 (scope3.com)","AmazonAdBot","yacybot"]}
{"domain":"ebay.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":15,"stanza_tokens":["*","bingbot","AdsBot-Google","AdIdxBot","Twitterbot","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","PerplexityBot","anthropic-ai","AmazonBot","meta-externalagent","GPTBot","Applebot-Extended"]}
{"domain":"cdiscount.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Mediapartners-Google","ia_archiver","AdsBot-Google","*"]}
{"domain":"justdial.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","ChatGPT-User/2.0","anthropic-ai","ClaudeBot","claude-web","PerplexityBot","Perplexity-User","Google-Extended","Googlebot","Bingbot","Amazonbot","Applebot","GPTBot","Baiduspider","Yandex","ia_archiver"]}
{"domain":"edna.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","Googlebot","*"]}
{"domain":"suumo.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","Googlebot-Mobile","Google-Extended","Googlebot","bingbot","msnbot","Crowsnest","Twitterbot"]}
{"domain":"megafon.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"tori.fi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ppomppu.co.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cpanel.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"faphouse2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"actu.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":19,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","CCBot","anthropic-ai","Claude-Web","ClaudeBot","Applebot-Extended","Amazonbot","Bytespider","cohere-ai","FacebookBot","MistralAI-User","MistralAI-Index","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","PerplexityBot"]}
{"domain":"ap4r.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"youporn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"polybuzz.ai","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"xhspot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"ladepeche.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":240,"stanza_tokens":["*","5emeRue","5erue","adequat","adequat-systems","Alexibot","AlvinetSpider","Amazonbot","AmiSoftware","Antenne Hatena","anthropic-ai","ApocalXExplorerBot","Argus","Ask n read","asknread.com","asterias","Augure","auramundi","BackDoorBot/1.0","BizInformation","Black Hole","Bloodhound","Bloomberg","BlowFish/1.0","BotALot","BraveBot","BuiltBotTough","Bullseye/1.0","BunnySlippers","Bytespider","CCBot","Cegbfeieh","ChatGPT-User","OAI-SearchBot","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","Cision","Claude-Web","ClaudeBot","coexel","cohere-ai","ConveraCrawler","CopyRightCheck","Corporama","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","cydralspider","Diffbot","Digimind","DISCo Pump 3.1","DittoSpyder","dotbot","Download Ninja","downloadexpress","EDD","ellisphere","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","eureka","Europresse","Explore","ExtractorPro","FacebookBot","Factiva","Fasterfox","Fetch","Flamingo_SearchEngine","flipboard","Foobot","gammaSpider","Google-Extended","GPTBot","grub-client","Harvest/1.5","hloader","httplib","HTTrack","HTTrack 3.0","humanlinks","ia_archiver","ia_archiver-web.archive.org","Igentia","indexer","InfoNaviRobot","infoseek","JennyBot","Jetbot","JikeSpider","k2spider","Kantar","kbcrawl","Kenjin Spider","Knowings","larbin","leadbox"]}
{"domain":"cbc.ca","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"omnitagjs.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"spamhaus.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"amd.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"dhl.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"emag.ro","fetch_ok":false,"usable":false,"reason":"other","status":511,"error":null,"attempts":[511,511,"TimeoutError"]}
{"domain":"tendawifi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gtm-a2b2.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"t-bank-app.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"SELF_SIGNED_CERT_IN_CHAIN","attempts":["SELF_SIGNED_CERT_IN_CHAIN","TimeoutError"]}
{"domain":"oneme.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"hik-connect.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"sotwe.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"admanmedia.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"xvideos-cdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","TimeoutError"]}
{"domain":"uci.edu","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"mmechocaptiveportal.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","TimeoutError"]}
{"domain":"bihar.gov.in","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","UND_ERR_CONNECT_TIMEOUT"]}
{"domain":"makemytrip.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"dns.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cam.ac.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"elsevier.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["FAST Enterprise Crawler 6 / Scirus","innosense/Nutch-1.0","Sogou web spider/4.0","Xenu Link Sleuth/1.3.8","discoverybot/2.0","YoudaoBot/1.0","Sogou web spider/3.0","*"]}
{"domain":"adfuse-ssp.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ilfattoquotidiano.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","msnbot","bingbot","Arianna","Arianna News","Arianna Web","Arianna Robot","Webnews Arianna","magpie-crawler"]}
{"domain":"rayjump.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"crn77.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"redtube.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"donmai.us","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"autotrader.co.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"fedex.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"haberler.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pornone.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"blackrussia.online","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"funpay.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AhrefsBot"]}
{"domain":"calculator.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"championat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Googlebot","Yandex","*"]}
{"domain":"thisvid.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Yandex","bingbot","SemrushBot","ia_archiver","ia_archiver-web.archive.org","MJ12bot","AhrefsBot","Seekport Crawler"]}
{"domain":"xhsocial.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"t-s1-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hollywoodbets.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sciencedaily.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"asurascans.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xxxbp.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["ia_archiver","*"]}
{"domain":"aternos.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"id.me","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"huggingface.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"9ffyu1g9c.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wayground.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"pornhat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"e621.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":26,"stanza_tokens":["GPTBot","CCBot","anthropic-ai","Claude-Web","Google-Extended","PerplexityBot","Bytespider","Amazonbot","Omgilibot","Meta-ExternalAgent","Diffbot","ImagesiftBot","cohere-ai","MJ12bot","dotbot","coccocbot-web","BLEXBot","HTTrack","Slurp","Baiduspider","Yandex","magpie-crawler","SemrushBot","SemrushBot-SA","AhrefsBot","*"]}
{"domain":"fdown.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"subito.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mediaexpert.pl","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tenki.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","CriteoBot","peer39_crawler","Leikibot","ias_crawler","ias_wombles","proximic","AudigentAdBot"]}
{"domain":"aznude.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xvv1deos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"publicnode.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fapello.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xnxx.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"youjizz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"weathernews.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["PerplexityBot","ClaudeBot","GPTBot","Google-Extended","*"]}
{"domain":"hitomi.la","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"telefonica.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"leroymerlin.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Nutch","*"]}
{"domain":"aljazeera.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":10,"stanza_tokens":["*","AmazonAdBot","anthropic-ai","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","GPTBot","PerplexityBot","Bytespider"]}
{"domain":"e-hentai.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"missav.ws","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ajio.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"navitime.co.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"imhentai.xxx","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"turkiye.gov.tr","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","ETIMEDOUT"]}
{"domain":"pornhub.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":17,"stanza_tokens":["*","Twitterbot","Amazonbot","Anthropic-ai","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","Cohere-ai","DataForSeoBot","ImagesiftBot","Magpie-crawler","Omgili","Omgilibot","Peer39_crawler","Peer39_crawler/1.0","YouBot"]}
{"domain":"rule34video.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cardgames.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"pulse.qa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sun.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"wisc.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mangabuff.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sportybet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"akismet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ixxx.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"missav.live","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jable.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"glasvezelactiveren.nl","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"bef77.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"mbga.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot-Mobile","Y!J-SRD/1.0","Twitterbot","Slurp"]}
{"domain":"mangago.me","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"amap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":49,"stanza_tokens":["Baiduspider","Baiduspider-image","Googlebot","MSNBot","bingbot","Sogou web spider","Sogou inst spider","Sogou spider2","Sogou blog","Sogou News Spider","Sogou Orion spider","360Spider","YisouSpider","YoudaoBot","ChinasoSpider","Sosospider","EasouSpider","JikeSpider","*","GPTBot","ChatGPT-User","Claude-Web","Anthropic-AI","PerplexityBot","Google-Extended","Applebot","Bytespider","QwenBot","TongyiBot","DeepSeekBot","DeepSeek-User","Kimi","MoonshotBot","ChatGLM-Spider","Hunyuan","TencentBot","MiniMaxBot","OAI-SearchBot","Doubao","AliyunBot","PanguBot","iFlytekBot","YiBot","SenseBot","Amazonbot","YouBot","Cohere-AI","Meta-ExternalAgent","CCBot"]}
{"domain":"boursorama.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"boardgamearena.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"umn.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":[""]}
{"domain":"maricopa.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Baiduspider","Yandex","*","Siteimprove","Siteimprovebot","Siteimprovebot-crawler"]}
{"domain":"ixl.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":4,"stanza_tokens":["*","GPTBot","CCBot","Applebot-Extended"]}
{"domain":"varzesh3.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"naukri.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"dagbladet.no","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":17,"stanza_tokens":["*","Amazonbot","Applebot","Applebot-Extended","Bytespider","CCBot","Claude-User","ClaudeBot","Claude-SearchBot","anthropic-ai","Google-Extended","GPTBot","ChatGPT-User","cohere-ai","meta-externalagent","PerplexityBot","OAI-SearchBot"]}
{"domain":"idealista.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bookmyshow.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Applebot","Googlebot","PetalBot","*"]}
{"domain":"attn.tv","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ECONNRESET",404]}
{"domain":"sex-studentki.team","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"apple-cloudkit.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"18comic.vip","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","Googlebot","bingbot","SemrushBot","dotbot","AhrefsBot","YandexBot","MJ12bot","BlexBot","Baiduspider","Sogou","PetalBot","CCBot","MauiBot","serpstatbot","SEOkicks-Robot"]}
{"domain":"58.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thexh.live","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"gst.gov.in","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"netkeiba.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"fanfiction.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"buydomains.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"simpcity.cr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"novinky.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","SeznamSocialBot","AI"]}
{"domain":"ilmeteo.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Googlebot","008","MJ12bot","Nutch","Riddler","*"]}
{"domain":"coolmathgames.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jcphi.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"platorelay.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"deltafibernederland.nl","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"syosetu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Mediapartners-Google","BingBot","meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)","meta-externalagent/1.1"]}
{"domain":"carsensor.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zozo.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ECONNRESET","TimeoutError"]}
{"domain":"bytefcdn-ttpeu.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"porno365.golf","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"dantri.com.vn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"uidai.gov.in","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"beboo.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cloudflareclient.com","fetch_ok":false,"usable":false,"reason":"other","status":522,"error":null,"attempts":[522,"ENOTFOUND",522]}
{"domain":"gcash.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"economictimes.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"eenadu.net","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":["TimeoutError",202]}
{"domain":"cronista.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zedge.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":8,"stanza_tokens":["*","turnitinbot","NPBot","ia_archiver","webzip","OmniExplorer_Bot","BoardReader","GPTBot"]}
{"domain":"iltalehti.fi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":70,"stanza_tokens":["*","AI2Bot","Ai2Bot-Dolma","AliyunSecBot","Amazonbot","AmazonBot","anthropic-ai","Applebot-Extended","AudigentAdBot","AwarioRssBot","AwarioSmartBot","BLEXBot","Bytespider","CCBot","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","DataForSeoBot","DeepSeek","DeepSeekBot","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","GPTBot","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Jetslide","Kangaroo Bot","magpie-crawler","Meta-ExternalAgent","meta-externalagent","Meta-ExternalFetcher","meta-externalfetcher","Meta-WebIndexer","meta-webindexer/1.1","MistralAI-User","MyCentralAIScraperBot","news-please","NewsNow","OAI-SearchBot","omgili","omgilibot","PanguBot","peer39_crawler","peer39_crawler/1.0","Perplexity-User","PerplexityBot","Poseidon Research Crawler","quillbot.com","Quora-Bot","Qwam content intelligence","Scrapy","SeekrBot","SeznamHomepageCrawler","Sidetrade indexer bot","TaraGroup Intelligent Bot","TeleportPro","Timpibot","TurnitinBot","VelenPublicWebCrawler","ViennaTinyBot","Webzio-Extended","YouBot"]}
{"domain":"jalan.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"blooket.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"index.hr","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tinkoff.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","Yandex","Googlebot-image","Googlebot","Googlebot-Mobile","AdsBot-Google","AdsBot-Google-Mobile","GPTBot","Bingbot","ia_archiver","YandexAdditionalBot"]}
{"domain":"tbank.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","Yandex","Googlebot-image","Googlebot","Googlebot-Mobile","AdsBot-Google","AdsBot-Google-Mobile","GPTBot","Bingbot","ia_archiver","YandexAdditionalBot"]}
{"domain":"inps.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"stratoserver.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"3bmeteo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cardmarket.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"arca.live","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"immobiliare.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cityheaven.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"jagran.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"myinstants.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"deepseek.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"fpo.xxx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dbankcloud.cn","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"property24.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","trovitBot"]}
{"domain":"eff.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["SemrushBot","AhrefsBot","MJ12bot","GPTBot","Google-Extended","CCBot","anthropic-ai","ClaudeBot","facebookexternalhit","*"]}
{"domain":"ficbook.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gamewith.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"sexvid.xxx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["BingBot","*"]}
{"domain":"ssstik.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ukdevilz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"superporn.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"uniqlo.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"kaspi.kz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","CazoodleBot","MJ12bot","dotbot/1.0","Gigabot"]}
{"domain":"pussyspace.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"fuq.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"ruliweb.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"leroymerlin.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":154,"stanza_tokens":["*","Googlebot","Googlebot-News","Googlebot-Image","Googlebot-Video","Googlebot-Mobile","AdsBot-Google","008","Alexibot","Aqua_Products","b2w/0.1","BackDoorBot/1.0","Bookmark search tool","BotALot","BotRightHere","BuiltBotTough","Bullseye/1.0","BunnySlippers","CherryPicker","CherryPickerSE/1.0","CherryPickerElite/1.0","CheeseBot","CNCDialer","Copernic","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","DittoSpyder","DOC","Download Ninja","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","ExtractorPro","FairAd Client","fast","Fetch","Flaming AttackBot","Foobot","Gaisbot","GetRight/4.2","grub-client","Harvest/1.5","hloader","httplib","HTTrack","HTTrack 3.0","humanlinks","InfoNaviRobot","Iron33/1.0.2","JennyBot","JikeSpider","Jyxobot/1","k2spider","Kenjin Spider","Keyword Density/0.9","larbin","LexiBot","libWeb/clsHTTP","libwww","LinkextractorPro","linko","LinkScan/8.1a Unix","LinkWalker","LNSpiderguy","lwp-trivial/1.34","lwp-trivial","Mata Hari","Maxthon","MIIxpc","MIIxpc/4.2","Mister PiX","moget","moget/2.1","NetAnts","NICErsPRO","NPBot","Offline Explorer","Openbot","Openfind data gatherer","Openfind","Oracle Ultra Search","PerMan","ProWebWalker","ProPowerBot/2.14","Python-urllib","QueryN Metasearch","Radiation Retriever 1.1","RepoMonkey Bait & Tackle/v1.01","RepoMonkey","RMA","searchpreview","sitecheck.internetseer.com","SiteSnagger","SpankBot","spanner","suzuran","Szukacz/1.4","Teleport"]}
{"domain":"ekstrabladet.dk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":35,"stanza_tokens":["AhrefsBot","AhrefsSiteAudit","DataForSeoBot","Eyeotabot","Mediatoolkitbot","msnbot","SemrushBot","SeznamBot","Yandex","Amazonbot","anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","CCBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-Web","Claude-SearchBot","DeepSeekBot","DeepSeek","Diffbot","DuckAssistBot","Google-Extended","GPTBot","meta-externalagent","meta-externalfetcher","OAI-SearchBot","PerplexityBot","Perplexity-User","Timpibot","YouBot","Mediapartners-Google","*"]}
{"domain":"sxyprn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"forebet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"luxuretv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"letsporn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"klikbca.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cmoa.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","Twitterbot","AhrefsBot","SemrushBot","DotBot","MJ12bot","serpstatbot","SEOkicks","Barkrowler","BLEXBot","SEBot-WA"]}
{"domain":"hdtube.porn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":7,"stanza_tokens":["BingBot","*","GPTBot","ChatGPT-User","CCBot","PerplexityBot","Google-Extended"]}
{"domain":"theblowers.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"supersonicads.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"starfall.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xnxx.health","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"1drv.com","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",400]}
{"domain":"caliente.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"datadome.co","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"aka.ms","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rat.xxx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xvideos.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"toyhou.se","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"nosv.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["Googlebot"]}
{"domain":"mangafire.to","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"255md.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"game8.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":3,"stanza_tokens":["Googlebot","dotbot","GPTBot"]}
{"domain":"xnxx.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xnxx2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"russianpost.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ioh.co.id","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"omg10.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UND_ERR_CONNECT_TIMEOUT","attempts":["UND_ERR_CONNECT_TIMEOUT","TimeoutError"]}
{"domain":"karnataka.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nesine.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"eltiempo.es","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ovhcloud.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"newsweek.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"sozcu.com.tr","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"otomoto.pl","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"betway.co.za","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"testbook.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":17,"stanza_tokens":["*","IstellaBot","Curious George","ZoomBot","TurnitinBot","Scanbot","oodlebot","rtbtr-adstxt-bot","linkdexbot","Mail.RU_Bot","musobot","YaK","TweetmemeBot","James BOT","trendictionbot","Mediatoolkitbot","SurdotlyBot"]}
{"domain":"s8ey.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"24h.com.vn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"blic.rs","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"theqoo.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"wshifen.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"protothema.gr","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"jutarnji.hr","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"imagefap.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","UND_ERR_CONNECT_TIMEOUT","TimeoutError"]}
{"domain":"rustdesk.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"wpastra.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","GPTBot","ClaudeBot","Google-Extended","CCBot","anthropic-ai"]}
{"domain":"exhentai.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"contextweb.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"bookmark.xxx","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"123av.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"fatalmodel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xvideos2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kooora.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sondakika.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"missav.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eci.gov.in","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"missav123.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hanime1.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mat6tube.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"joyclub.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":2,"stanza_tokens":["*","GPTBot"]}
{"domain":"wnacg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"markt.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":31,"stanza_tokens":["*","AdsBot-Google","Mediapartners-Google","ClaudeBot","CrystalSemanticsBot","DOC","Download Ninja","Fetch","grub-client","HTTrack","k2spider","larbin","libwww","linko","Microsoft.URL.Control","MSIECrawler","NPBot","Offline Explorer","sitecheck.internetseer.com","SiteSnagger","Teleport","TeleportPro","UbiCrawler","WebCopier","WebReaper","WebStripper","WebZIP","Xenu","Zao","Zealbot","ZyBORG"]}
{"domain":"porn300.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"typing.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"supjav.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"xnxx-arabic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"goodreturns.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","Mediapartners-Google","Googlebot-News","grapeshot","AmazonAdBot","ClaudeBot","anthropic-ai","GPTBot","ChatGPT-User","Google-Extended","PerplexityBot","CCBot"]}
{"domain":"bt.dk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":3,"stanza_tokens":["GPTBot","CCBot","*"]}
{"domain":"snaptik.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","magicsearchbot"]}
{"domain":"programme-tv.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["*","Mediapartners-Google","Googlebot-News","CCBot","Meltwater","trendictionbot","trendictionbot0.5.0","trendiction search","um-IC","um-FC","Panalis","LexisNexis","PetalBot"]}
{"domain":"auth0.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"indianrail.gov.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fazenda.gov.br","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"mps.gov.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"jennymovies.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"listcrawler.eu","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"tabletki.ua","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"eepurl.com","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":[400,"TimeoutError"]}
{"domain":"tdnsstic1.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"tokyomotion.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aajtak.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot-News"]}
{"domain":"sexlog.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Googlebot","Googlebot-Image","Mediapartners-Google","Adsbot-Google"]}
{"domain":"shutterstock.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":11,"stanza_tokens":["*","CCBot","GPTBot","ClaudeBot","meta-externalagent","OAI-SearchBot","PerplexityBot","Claude-SearchBot","adidxbot","bingbot","Yandex"]}
{"domain":"google.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"casinoplus.com.ph","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mechacomic.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lioden.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":2,"stanza_tokens":["dotbot","GPTBot"]}
{"domain":"bikewale.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AhrefsBot","MJ12bot","*"]}
{"domain":"fastdl.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"therapservices.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iporntv.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"ptt.cc","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ECONNRESET","ECONNREFUSED"]}
{"domain":"fedoraproject.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"site.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"croxyproxy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iplt20.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"myreadingmanga.info","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"qorno.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"bingoplus.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"twpornstars.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"serviporno.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Amazonbot","*"]}
{"domain":"nya2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"adrta.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"trilltrill.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AmazonAdBot","Twitterbot","*","AdsBot-Google-Mobile","AdsBot-Google","Mediapartners-Google"]}
{"domain":"kompoz2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":26,"stanza_tokens":["*","YandexVideoParser","008","BLEXBot","AhrefsBot","Vagabondo","SEOkicks-Robot","ia_archiver","archive.org_bot","special_archiver","MJ12bot","heritrix","netEstate NE Crawler","sistrix","WBSearchBot","QuerySeekerSpider","proximic","SiteExplorer","SemrushBot","SemrushBot-SA","Mail.Ru","Baiduspider","MSNBot","Bingbot","Slurp","Yandex"]}
{"domain":"ey43.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"6opo.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"dogdrip.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"hochi.news","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":29,"stanza_tokens":["Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","cohere-training-data-crawler","Diffbot","FacebookBot","Google-Extended","GPTBot","ICC-Crawler","magpie-crawler","meta-externalagent","meta-externalfetcher","OAI-SearchBot","omgili","omgilibot","PerplexityBot","Perplexity-User","Scrapy","Timpibot","Webzio-Extended","Gensparkbot"]}
{"domain":"nickfinder.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"piccoma.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ladies.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Chrome-Lighthouse","*"]}
{"domain":"ijavhd.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"skokka.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"kingbokep.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"alura.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"siemens.com","fetch_ok":false,"usable":false,"reason":"other","status":405,"error":null,"attempts":[405,405]}
{"domain":"chordtela.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["grapeshot","*"]}
{"domain":"screener.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","ETIMEDOUT"]}
{"domain":"epfindia.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ak2yy.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"betika.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["facebookexternalhit","Twitterbot","*"]}
{"domain":"rkv1.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"xorgasmo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pussyboy.net","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"okxxx1.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hentai.name","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":26,"stanza_tokens":["*","008","BLEXBot","AhrefsBot","Vagabondo","SEOkicks-Robot","ia_archiver","archive.org_bot","special_archiver","MJ12bot","heritrix","netEstate NE Crawler","sistrix","WBSearchBot","QuerySeekerSpider","proximic","SiteExplorer","SemrushBot","SemrushBot-SA","DotBot","Mail.Ru","Baiduspider","MSNBot","Bingbot","Slurp","Yandex"]}
{"domain":"librus.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"syosetu.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"overwolf.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"fao.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bakusai.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["ia_archiver","Megalodon"]}
{"domain":"girlschannel.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xxxvideo.link","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sexalarab.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"your-server.de","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"afip.gob.ar","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",404]}
{"domain":"ss.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"goojara.to","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"njavtv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ftc.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"search-start.org","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"freepornvideo.sex","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"desitales2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"moviebox.ph","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot","Bingbot","Slurp"]}
{"domain":"spaggiari.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xvideos-ar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rakuten-sec.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"arabxn.sex","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fabswingers.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"skelbiu.lt","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"imgsrc.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Yandex","MJ12bot","Baiduspider"]}
{"domain":"diretta.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"nustargame.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","GPTBot","ChatGPT-User","OAI-SearchBot","PerplexityBot","ClaudeBot","anthropic-ai","Google-Extended","Applebot","Amazonbot","Bytespider","cohere-ai"]}
{"domain":"pornocarioca.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vymanga.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"an1.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"redwap.sex","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":26,"stanza_tokens":["*","YandexVideoParser","008","BLEXBot","AhrefsBot","Vagabondo","SEOkicks-Robot","ia_archiver","archive.org_bot","special_archiver","MJ12bot","heritrix","netEstate NE Crawler","sistrix","WBSearchBot","QuerySeekerSpider","proximic","SiteExplorer","SemrushBot","SemrushBot-SA","Mail.Ru","Baiduspider","MSNBot","Bingbot","Slurp","Yandex"]}
{"domain":"lausd.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"sex-studentki.cafe","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nfse.gov.br","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"apdcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"banvenez.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"arabx.cam","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":12,"stanza_tokens":["*","Amazonbot","Bytespider","CCBot","ClaudeBot","GPTBot","meta-externalagent","anthropic-ai","YouBot","cohere-ai","Diffbot","ImagesiftBot"]}
{"domain":"happymh.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"momon-ga.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"lk21official.cc","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"newyorker.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","FacebookExternalHit","Twitterbot","Pinterestbot","LinkedInBot","TikTokSpider","Storebot-Google","Google-InspectionTool","AmazonAdBot","Google-Extended","Google-CloudVertexBot","GoogleOther","Applebot-Extended","Amazonbot","meta-webindexer","meta-externalagent","meta-externalfetcher","ClaudeBot","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","cohere-training-data-crawler","cohere-ai","CCBot","PanguBot","PetalBot","Bytespider","Diffbot","DuckAssistBot","MistralAI-User","Timpibot","Webzio","Webzio-Extended","archive.org_bot","ia_archiver","ia_archiver-web.archive.org","heritrix"]}
{"domain":"sexnyk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ultradns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"absher.sa","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"parivahan.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"tktube.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"agenciatributaria.gob.es","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"motherless.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"qlikdev.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"czbooks.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"kakuyomu.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"sinoptik.ua","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ptgaming.ph","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"redbus.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"choiceadvantage.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"bancodevenezuela.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","attempts":["ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","TimeoutError"]}
{"domain":"publi24.ro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":22,"stanza_tokens":["GPTBot","CCBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","claude-web","anthropic-ai","Googlebot","Google-Extended","PerplexityBot","Applebot","Applebot-Extended","DeepSeek","WangluTechBot","Baiduspider","360Spider","Sogou Spider","YisouSpider","Bytespider","YoudaoBot","ToutiaoSpider","*"]}
{"domain":"keirin.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tjk.org","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":["TimeoutError",400]}
{"domain":"yallakora.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"alphapolis.co.jp","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":["TimeoutError",202]}
{"domain":"porno666.fo","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ECONNRESET","TimeoutError"]}
{"domain":"promiedos.com.ar","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"xosodaiphat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"flashscore.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"afribaba.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"jra.go.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zebawy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Mediapartners-Google*","Googlebot-Image"]}
{"domain":"anses.gob.ar","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"arbada.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"letras.mus.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"arabtnt.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"impots.gouv.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mangabuddy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"manhwaweb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"mileroticos.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"xxx18hot.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","TimeoutError"]}
{"domain":"indown.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sexemodel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jra.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hotxv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sarkariresult.com.cm","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xvideosputaria.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"avito.st","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"twkan.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"flashscore.mobi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"pornhat.one","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"xpaja.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot","Googlebot-Video"]}
{"domain":"ewaybillgst.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"rfihub.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"dto.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"betking.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"animanch.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"animesalt.ac","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"jaya9bangladesh.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["Rogerbot","Exabot","MJ12bot","Dotbot","Gigabot","SemrushBot","SemrushBot-SA","Semrushbot-SI","*"]}
{"domain":"ojogodobicho.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Mediapartners-Google","AdsBot-Google","AdsBot-Google-Mobile"]}
{"domain":"conectate.com.do","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xsmn.mobi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"indiansexstories3.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"espn.com.ve","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":11,"stanza_tokens":["claritybot","GPTBot","Google-Extended","CCBot","ChatGPT-User","anthropic-ai","Omgilibot","Omgili","FacebookBot","Bytespider","*"]}
{"domain":"xoso.com.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"livescore.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"administracionelectronica.gob.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"az24.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"meteofor.com.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"darkero.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"skipthegames.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"autoplius.lt","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"turbo.az","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"patria.org.ve","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"braiiz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"triunfobet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aucey.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",404]}
{"domain":"smadex.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zijieapi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fabguys.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"humoruniv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":[],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Bytespider","GPTBot","ClaudeBot","Amazonbot","OAI-SearchBot","AliyunSecBot","Applebot","Meta-ExternalAgent","ia_archiver"]}
{"domain":"betpawa.ug","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","ClaudeBot","anthropic-ai","PerplexityBot","Amazonbot","Bytespider","cohere-ai"]}
{"domain":"crex.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":6,"stanza_tokens":["*","duggmirror","grapeshot","SemrushBot","AhrefsBot","GPTBot"]}
{"domain":"zearn.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"packzy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kumanga.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"w3schools.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Site24x7"]}
{"domain":"acesso.gov.pt","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"acm.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xxxhindi.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"pewresearch.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"koselarab.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"fifa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kemkes.go.id","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"escort.club","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"annunci69.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":130,"stanza_tokens":["*","BotRightHere","WebZip","larbin","b2w/0.1","Copernic","psbot","Python-urllib","URL_Spider_Pro","CherryPicker","EmailCollector","EmailSiphon","WebBandit","EmailWolf","ExtractorPro","CopyRightCheck","Crescent","SiteSnagger","ProWebWalker","CheeseBot","LNSpiderguy","Alexibot","Teleport","TeleportPro","MIIxpc","Telesoft","Website Quester","moget/2.1","WebZip/4.0","WebStripper","WebSauger","WebCopier","NetAnts","Mister PiX","WebAuto","TheNomad","WWW-Collector-E","RMA","libWeb/clsHTTP","asterias","httplib","turingos","spanner","InfoNaviRobot","Harvest/1.5","Bullseye/1.0","Mozilla/4.0 (compatible; BullsEye; Windows 95)","Crescent Internet ToolPak HTTP OLE Control v.1.0","CherryPickerSE/1.0","CherryPickerElite/1.0","WebBandit/3.50","NICErsPRO","Microsoft URL Control - 5.01.4511","DittoSpyder","Foobot","SpankBot","BotALot","lwp-trivial/1.34","lwp-trivial","BunnySlippers","Microsoft URL Control - 6.00.8169","URLy Warning","Wget/1.6","Wget/1.5.3","Wget","LinkWalker","cosmos","moget","hloader","humanlinks","LinkextractorPro","Offline Explorer","Mata Hari","LexiBot","Web Image Collector","The Intraformant","True_Robot/1.0","True_Robot","BlowFish/1.0","JennyBot","MIIxpc/4.2","BuiltBotTough","ProPowerBot/2.14","BackDoorBot/1.0","toCrawl/UrlDispatcher","WebEnhancer","suzuran","TightTwatBot","VCI WebViewer VCI WebViewer Win32","VCI","Szukacz/1.4","QueryN Metasearch","Openfind data gatherer","Openfind","Xenu's Link Sleuth 1.1c","Xenu's","Zeus","RepoMonkey Bait & Tackle/v1.01","RepoMonkey","Microsoft URL Control"]}
{"domain":"tamildhool.tech","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"scan-manga.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Googlebot-Image","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"sattamatkadpboss.mobi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot","Googlebot-Image","Bingbot"]}
{"domain":"nz.ua","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"oreilly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":65,"stanza_tokens":["*","Baiduspider","PetalBot","yisouspider","008","ChatGPT-User","ChatGPT-User/2.0","PerplexityBot","Perplexity-User","DuckAssistBot","Applebot","bingbot","Googlebot","ClaudeBot","claude-web","Claude-SearchBot","YouBot","Applebot-Extended","MistralAI-User","meta-webindexer","GoogleAgent-Mariner","DotBot","archive.org_bot","Pinterestbot","TelegramBot","kakaotalk-scrap","getstream.io/opengraph-bot","RamblerMail","Brave","OAI-SearchBot","GPTBot","anthropic-ai","Google-Extended","cohere-ai","CCBot","AI2Bot","Amazonbot","Amazonbot-Video","Bytespider","meta-externalagent","Diffbot","omgili","TimpiBot","SeznamBot","Exabot","YandexBot","Sogou","360Spider","YisouSpider","Baiduspider-Render/2.0","MJ12bot","DataForSeoBot","AliyunSecBot","ArchiveTeam","ArchiveTeam ArchiveBot","AwarioBot","ZoominfoBot","ali-implementer","Blueno","BIGO-baiguoyuan","WorksOgCrawler","OnPageBot","DoCoMo","SAMSUNG-SGH-E250","TA-Googlebot"]}
{"domain":"limo.media","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kinnser.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"betpawa.co.tz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","ClaudeBot","anthropic-ai","PerplexityBot","Amazonbot","Bytespider","cohere-ai"]}
{"domain":"unlockffbeta.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"battle.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"gree-apps.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fojik.site","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["ia_archiver","*"]}
{"domain":"mobiledokan.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oolveri.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"loteriadehoy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"bpexch.live","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"bhojpurisex.site","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"oddspark.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bpcl.in","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kkstories.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"emaktab.uz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","*","BLEXBot"]}
{"domain":"maynoe.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eroterest.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","UND_ERR_CONNECT_TIMEOUT","ENOTFOUND"]}
{"domain":"blacktoon410.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"xbanxia.cc","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mobcup.fm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":25,"stanza_tokens":["Bytespider","DataForSeoBot","sitebot","Riddlerbot","Ocelli","linguatools","www.integromedb.org/Crawler","008","Exabot","psbot","turnitinbot","NPBot","ia_archiver","webzip","OmniExplorer_Bot","BoardReader","GPTBot","Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News","Mediapartners-Google","AdsBot-Google","Google-Display-Ads-Bot","*"]}
{"domain":"tjsp.jus.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lotterysambadresult.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot","Googlebot-Image"]}
{"domain":"xn--72c0anj1fqy6jqa7ei.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"satta-king-fast.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"wetter.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","RyteBot","Mediapartners-Google","AmazonAdBot","DoubleVerifyBot","DVBot","Leikibot"]}
{"domain":"sattamatkano1.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"antarvasna3.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"freesexkahani.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"desitelugusex.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ad-contents.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["archive.org_bot","MJ12bot","Googlebot","bingbot"]}
{"domain":"vnsextop1.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"shopee.com.my","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot","Googlebot-Mobile","Bingbot","*"]}
{"domain":"hpeprint.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",400]}
{"domain":"playinrace.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"playinboss.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"salonboard.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"xxxvideosind.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"rsch.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"onlar.az","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"site777.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"dzexams.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":59,"stanza_tokens":["*","Mediapartners-Google*","Mediapartners-Google","sistrix","SISTRIX Crawler","SISTRIX","SEOkicks-Robot","jobs.de-Robot","AhrefsBot","UnisterBot","SearchmetricsBot","MJ12bot","SurveyBot","SEOdiver","spbot","wotbox","dotbot","meanpathbot","BacklinkCrawler","magpie-crawler","oBot","fr-crawler","BLEXBot","MegaIndex.ru","megaindex.com","CloudServerMarketSpider","trendictionbot","Exabot","careerbot","Lipperhey-Kaus-Australis","seoscanners.net","MetaJobBot","Spiderbot","LinkStats","JobboerseBot","ICCrawler","Plista","Domain Re-Animator Bot","turnitinbot","coccoc","um-IC","mindUpBot","sg-Orbiter","CCBot","Qwantify","YaK","plukkie","SafeDNSBot","360Spider","HaosouSpider","rogerbot","OpenHoseBot","Screaming Frog SEO Spider","ThumbSniper","R6_CommentReader","ImplisenseBot","Cliqzbot","aiHitBot","WBSearchBot"]}
{"domain":"goal7.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jino.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"staseraintv.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"dpboss.boston","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"simontokx.asia","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"gree-pf.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bunnyband.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"fawanews.sc","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"xnhau.ing","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kargo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":30,"stanza_tokens":["AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","DuckAssistBot","FacebookBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","img2dataset","Meta-ExternalAgent","MyCentralAIScraperBot","omgili","omgilibot","Quora-Bot","TikTokSpider","YouBot","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","*"]}
{"domain":"funinstar.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[405,202]}
{"domain":"immersivelabs.online","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"acs.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"ecnavi.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TypeError"]}
{"domain":"atcpulsa.store","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"cmu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sdpay.co.in","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"azure-api.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"robinhood.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sslip.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"trustarc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ctdns.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"marketwatch.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":72,"stanza_tokens":["*","googlebot","googlebot-image","GoogleOther","Googlebot-Video","Google-InspectionTool","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","Storebot-Google","google-read-aloud","mediapartners-google","bingbot","msnbot","bingpreview","slurp","yahoo","yandex","YandexImages","baiduspider","Pinterestbot","Yeti","MojeekBot","360Spider","google-cloudvertexbot","duckduckbot","petalbot","Applebot","AppleNewsBot","flipboard","qwantbot","SeznamBot","yisouspider","proximic","admantx","thetradedesk","outbrain","ias_crawler","AmazonAdBot","pubmatic","smartologybot","parselybot","Screaming Frog SEO Spider","DeepCrawl","AhrefsBot","SemrushBot","SimilarWebBot","SISTRIX","OnCrawl","botify","Chrome-Lighthouse","ChatGPT-User","GPTBot","OAI-SearchBot","facebookexternalhit","facebot","twitterbot","linkedinbot","telegrambot","snapchat","newrelicpinger","siteimprovebot","uptimerobot","pingdom.com_bot_version_1.4","sentry","schema-validator","googleimageproxy","Iframely","Vocabtracker","EpvzCrawl6194680250","Citoid","ZoteroTranslationServer"]}
{"domain":"inleed.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"instructure.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ico.org.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","deepcrawl"]}
{"domain":"moodle.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AhrefsBot","*"]}
{"domain":"madrasati.sa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"jamfcloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"ad-tech.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","TimeoutError"]}
{"domain":"api.aws","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"mgslb.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","UND_ERR_CONNECT_TIMEOUT"]}
{"domain":"usercontent.microsoft","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"amazon.eg","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UND_ERR_CONNECT_TIMEOUT","attempts":["UND_ERR_CONNECT_TIMEOUT","TimeoutError"]}
{"domain":"fb.watch","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"scientificamerican.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":12,"stanza_tokens":["008","*","CCBot","GPTBot","ChatGPT-User","anthropic-ai","cohere-ai","Google-Extended","PerplexityBot","RepoLookupBot","AwarioSmartBot","PerplexityAI"]}
{"domain":"nps.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","bingbot"]}
{"domain":"va.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["usasearch","Synapse","Amazonbot","*"]}
{"domain":"adroll.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xham.live","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","ClaudeBot","Amazonbot","facebookexternalhit"]}
{"domain":"uchicago.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xinhuanet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vmware.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zdnet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":113,"stanza_tokens":["*","msnbot","AddSearchBot","AI2Bot","Ai2Bot-DeepResearchEval","Ai2Bot-Dolma","Amazonbot","AmazonBuyForMe","Amzn-SearchBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot","Applebot-extended","atlassian-bot","AutoRAG","AwarioSmartBot","AzureAI-SearchBot","Big Sur AI","bigsur.ai","Brandwatch","Bravebot","Brightbot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","Datenbank-Crawler","DeepSeekBot","Devin","Diffbot","Direqt Anomura","DuckAssistBot","Echobot Bot","ExaBot","FacebookBot","Factset_spyderbot","FirecrawlAgent","GPTBot","iAsk","iAskBot","iaskspider","ICC Crawler","ICC-Crawler","ImageSiftBot","imageSpider","Kangaroo Bot","Kangaroo-Bot","KlaviyoAIBot","Kunato","laion-huggingface-processor","LCC","LINER Bot","LinerBot","LinkupBot","Manus-User","meta-externalagent","meta-externalfetcher","meta-webindexer","mistral.ai","MistralAI-User","netEstate Imprint Crawler","NovaAct","Novellum","Novellum AI Crawl","OAI-SearchBot","omgili","opencode","PanguBot","PeopleInc-DCipher-Scraper/1.1.0","Perplexity-User","PerplexityBot","PetalBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","SeekrBot","SemrushBot-OCOB","SemrushBotSwa","ShapBot","Spider","TavilyBot","TerraCotta","Timpibot","Trae","TwinAgent","Velen Crawler","VelenPublicWebCrawler","Webzio-Extended"]}
{"domain":"wikihow.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":76,"stanza_tokens":["Ai2Bot","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","archive.org","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","Diffbot","DOC","Download Ninja","DuckAssistBot","FacebookBot","Facebookexternalhit","Fetch","Firecrawl","FirecrawlAgent","FriendlyCrawler","GPTBot","GrokApp","HMSE_Robot","HTTrack","ia_archiver","ICC-Crawler","ImagesiftBot","img2dataset","k2spider","larbin","libwww","linko","Meta-ExternalAgent","Meta-ExternalFetcher","Meta-WebIndexer","Microsoft.URL.Control","MistralAI-User","MSIECrawler","NPBot","OAI-SearchBot","Offline Explorer","omgili","OmigiliBot","PerplexityBot","PetalBot","Scrapy","Seekr","ShapBot","sitecheck.internetseer.com","SiteSnagger","Teleport","TeleportPro","TikTokSpider","Timpibot","UbiCrawler","VelenPublicWebCrawler","WebCopier","WebReaper","WebStripper","Webzio-Extended","WebZIP","wget","Xenu","YouBot","Zao","Zealbot","ZyBORG","AdsBot-Google","Mediapartners-Google","Googlebot","*"]}
{"domain":"secureserver.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zpath.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gpcloudservice.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"arstechnica.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":37,"stanza_tokens":["Googlebot-Image","Mediapartners-Google*","Google-Extended","Google-CloudVertexBot","GoogleOther","Applebot-Extended","meta-webindexer","meta-externalagent","meta-externalfetcher","ClaudeBot","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","cohere-training-data-crawler","cohere-ai","CCBot","PanguBot","PetalBot","Bytespider","Diffbot","DuckAssistBot","MistralAI-User","Timpibot","Webzio","Webzio-Extended","archive.org_bot","ia_archiver","ia_archiver-web.archive.org","heritrix","anthropic-ai","Claude-Web","FacebookBot","Omgilibot","YouBot","Amazonbot","*"]}
{"domain":"pubmnet.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"udemy.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"statuspage.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"xfinity.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pluto.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":54,"stanza_tokens":["AdsBot-Google","AdsBot-Google-Mobile","Mediapartners-Google","adidxbot","AdIdxBot","OAI-AdsBot","AmazonAdBot","ChatGPT-User","Claude-User","Perplexity-User","Amzn-User","Google-Agent","meta-externalads","facebookexternalhit","FacebookBot","Slackbot-LinkExpanding","Slack-ImgProxy","Google-Read-Aloud","AudigentAdBot","Criteo","CriteoBot","Discordbot","GoogleMessages","Google-NotebookLM","Google-PageRenderer","Google-Adwords-Instant","Google-Adwords-Instant-Mobile","Google-Display-Ads-Bot","LinkedInBot","meta-externalfetcher","Pinterestbot","redditbot","TaboolaBot","TelegramBot","Twitterbot","Zemanta","yahoo-ad-monitoring","Bluesky","Cardyb","TikTokSpider","Amazon-Advertising-ad-standards-bot","outbrain","Quantcastbot","Google-Pinpoint","GoogleDocs","SkypeUriPreview","Yahoo Link Preview","AddThis","bitlybot","BufferLinkPreviewBot","Snapchat","adbeat-bot","AdkernelTopicCrawler","*"]}
{"domain":"sciencemag.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":8,"stanza_tokens":["*","Google-Extended","anthropic-ai","Twitterbot","GPTBot","ChatGPT-User","CCBot","Applebot-Extended"]}
{"domain":"unrulymedia.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"mundodeportivo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":25,"stanza_tokens":["UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","ClaudeBot","AmazonAdBot","*"]}
{"domain":"samsungknox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Twitterbot","facebookexternalhit"]}
{"domain":"divar.ir","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pusher.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"avira.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","GPTBot","Perplexity-User","PerplexityBot","Claude-User","ClaudeBot"]}
{"domain":"utexas.edu","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"livechatinc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","mauibot"]}
{"domain":"gtld.biz","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gdemoideti.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iana.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"qcloud.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sitescout.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"douyin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":26,"stanza_tokens":["Googlebot","Google-Extended","Bingbot","DuckDuckBot","Baiduspider","360Spider","Sogouspider","Yisouspider","ByteSpider","ToutiaoSpider","Sogou web spider","Sogou wap spider","Sogou inst spider","Sogou blog","Sogou News Spider","Sogou Orion Spider","Sosospider","PerplexityBot","Perplexity-User","OAI-SearchBot","GPTBot","ChatGPT-User","Sogou spider2","HaosouSpider","HaosoSpider","deepseekbot"]}
{"domain":"meitu.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"fastcompany.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":27,"stanza_tokens":["*","Googlebot","msnbot","bingbot","Twitterbot","Mediapartners-Google","Google-Extended","ScoutJet","Yandex","baiduspider","DuckDuckBot","BomboraBot","grapeshot","ccbot","GPTBot","Bard","Jasper","YouChat","Perplexity","Claude","Claude-Web","Neeva","Copy.ai","Writesonic","Anyword","AI21","meta-externalagent"]}
{"domain":"examinationservices.nic.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nregastrep.nic.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"joinindianarmy.nic.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"mnregaweb4.nic.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"azure.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"sinaedge.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ns1p.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"trellix.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bbb.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["facebookexternalhit","*"]}
{"domain":"vice.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thelancet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":22,"stanza_tokens":["*","Twitterbot","LinkedInBot","GPTBot","ChatGPT-User","CCBot","Google-Extended","PerplexityBot","AI2Bot","Amazonbot","anthropic-ai","ClaudeBot","Bytespider","cohere-training-data-crawler","cohere-ai","Diffbot","iAskBot/1.0","meta-externalagent","MistralAI-User","omgili","TimpiBot","YouBot"]}
{"domain":"rapid7.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gizmodo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["*","AmazonAdBot","AI2Bot","Bytespider","CCBot","Cohere-ai","Diffbot","FirecrawlAgent","ImagesiftBot","Kangaroo Bot","Meta-ExternalAgent","Omgilibot","PanguBot","Timpibot","Webzio-Extended","HTTrack","Nutch","Scrapy"]}
{"domain":"mathtag.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"enable-javascript.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"f95zone.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":[],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":59,"stanza_tokens":["GPTBot","ClaudeBot","Claude-User","Claude-SearchBot","CCBot","Google-Extended","Applebot-Extended","Facebookbot","Meta-ExternalAgent","Meta-ExternalFetcher","diffbot","PerplexityBot","Perplexity‑User","Omgili","Omgilibot","webzio-extended","ImagesiftBot","Bytespider","TikTokSpider","Amazonbot","Youbot","SemrushBot-OCOB","Petalbot","VelenPublicWebCrawler","TurnitinBot","Timpibot","OAI-SearchBot","ICC-Crawler","AI2Bot","AI2Bot-Dolma","DataForSeoBot","AwarioBot","AwarioSmartBot","AwarioRssBot","Google-CloudVertexBot","PanguBot","Kangaroo Bot","Sentibot","img2dataset","Meltwater","Seekr","peer39_crawler","cohere-ai","cohere-training-data-crawler","DuckAssistBot","Scrapy","Cotoyogi","aiHitBot","Factset_spyderbot","FirecrawlAgent","PetalBot","AspiegelBot","AhrefsBot","SemrushBot","DotBot","MauiBot","MJ12bot","proximic","*"]}
{"domain":"constantcontact.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"indexww.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"cqloud.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"edgecastdns.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","EAI_AGAIN"]}
{"domain":"geocities.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"elegantthemes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"intercom.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"litix.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"jamanetwork.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":40,"stanza_tokens":["008","*","AI2Bot","anthropic-ai","Claude-Web","ClaudeBot","Applebot-Extended","Bytespider","TikTokSpider","GPTBot","cohere-ai","cohere-training-data-crawler","CCBot","Cotoyogi","Ai2Bot-Dolma","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Google-Extended","PanguBot","ICC-Crawler","img2dataset","imgproxy","ISSCyberRiskCrawler","Kangaroo Bot","FacebookBot","meta-externalagent","Crawlspace","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","VelenPublicWebCrawler","omgili","omgilibot","Webzio-Extended","YouBot","aiHitBot","Brightbot 1.0","ImagesiftBot"]}
{"domain":"threads.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["Amazonbot","Applebot-Extended","Brightbot","ClaudeBot","Google-Extended","GPTBot","PerplexityBot","PetalBot","Scrapy","uptimerobot","viberbot","YaK","Yandex","Yeti","Applebot","Bingbot","Discordbot","DuckDuckBot","facebookexternalhit","Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News","Google-InspectionTool","LinkedInBot","msnbot","Pinterestbot","Screaming Frog SEO Spider","seznambot","Slurp","teoma","TelegramBot","Twitterbot","*"]}
{"domain":"google.co.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"deepintent.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"playhop.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Mediapartners-Google","AdsBot-Google"]}
{"domain":"blizzard.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"twilio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Swiftbot"]}
{"domain":"ipredictive.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"com.be","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"cafe24.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"yimg.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","ENOTFOUND",200]}
{"domain":"blueapron.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":4,"stanza_tokens":["*","GPTBot","HTTrack 3.0","HTTrack 3.0x"]}
{"domain":"firstvds.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","*"]}
{"domain":"ucsd.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"meteofrance.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dns-shop.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"ovh.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":6,"stanza_tokens":["*","GPTBot","ChatGPT-User","CCBot","anthropic-ai","TextBulkerBot"]}
{"domain":"tremorhub.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"mega.nz","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Twitterbot","facebookexternalhit","*"]}
{"domain":"restream-media.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"meetup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":2,"stanza_tokens":["*","GPTBot"]}
{"domain":"deezer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Twitterbot"]}
{"domain":"mobile.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Mediapartners-Google","Opebot-v (https://www.1plusx.com)","AmazonAdBot","*"]}
{"domain":"vivox.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"okdiario.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":31,"stanza_tokens":["*","ia_archiver","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","Slurp","Maxthon","CNCDialer","Googlebot","Googlebot-News","Googlebot-Image","Googlebot-Video"]}
{"domain":"sky.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["OAI-SearchBot","OAI-AdsBot","*","AdsBot-Google"]}
{"domain":"bitdefender.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"techtarget.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","MediaPartners-Google","Updownerbot","008","bingbot","msnbot"]}
{"domain":"hubapi.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","ENOTFOUND",200]}
{"domain":"rncdn7.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"walmart.com.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rustore.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xhscdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"lookmovie2.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"clevelandclinic.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AmazonAdBot","*"]}
{"domain":"oneindia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["*","Mediapartners-Google","Googlebot-News","grapeshot","AmazonAdBot","ClaudeBot","anthropic-ai","GPTBot","ChatGPT-User","Google-Extended","PerplexityBot","CCBot","AdsBot-Google"]}
{"domain":"themeisle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"www.nhs.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Ultraseek","AhrefsBot","Algolia Crawler","*","Googlebot"]}
{"domain":"plex.direct","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","EAI_AGAIN"]}
{"domain":"typepad.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"investing.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Mediapartners-Google","grapeshot","proximic","AmazonAdBot"]}
{"domain":"mapbox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["atlassian-bot","*"]}
{"domain":"queniuaa.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"igamecj.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wcdnga.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sendgrid.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"dnse1.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"supabase.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","GPTBot","ClaudeBot","Google-Extended","PerplexityBot"]}
{"domain":"staples.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"comix.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"shopifycloud.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"nikkansports.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":12,"stanza_tokens":["*","AmazonAdBot","CCBot","ChatGPT-User","GPTBot","Google-Extended","ICC-Crawler","anthropic-ai","Claude-Web","cohere-ai","omgili","omgilibot"]}
{"domain":"action.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"neal.fun","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"firebaseio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"digikala.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"salesforceliveagent.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["googlebot","*"]}
{"domain":"usgs.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fullxh.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"business.site","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"l-cdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"romarg.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"0123tt.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"in.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","EAI_AGAIN"]}
{"domain":"exelator.com","fetch_ok":false,"usable":false,"reason":"other","status":204,"error":null,"attempts":[204]}
{"domain":"mwscdn.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_SSL_TLSV1_UNRECOGNIZED_NAME"]}
{"domain":"samsungiotcloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vtwenty.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cloud.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ecdnx.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"aadrm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"spankbang.party","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"stripe.network","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"pornoborshch.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":2,"stanza_tokens":["GPTBot","*"]}
{"domain":"getbootstrap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"postrelease.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"waze.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["mauibot","semrushbot","ahrefsbot","blexbot","seo spider","Googlebot","*"]}
{"domain":"mozgcp.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"si.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"jhu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"matterport.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"globenewswire.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"moengage.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"plex.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"carwale.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["Baiduspider","Baiduspider-video","Baiduspider-image","Yandex","YandexBot","SemrushBot","MJ12bot","ZoominfoBot","AhrefsBot","*"]}
{"domain":"vlr.gg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bt.co","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"torob.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AhrefsBot"]}
{"domain":"cf.lk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"archive.is","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"about.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"166.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"beeline.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","Googlebot","*"]}
{"domain":"automattic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"acrobat.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"sweb.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"redditstatic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","Googlebot","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","Googlebot-Image","Googlebot-Video","Applebot","ClarityBot","bingbot","MSNBot-Media"]}
{"domain":"pnas.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":5,"stanza_tokens":["*","facebookexternalhit","LinkedInBot","Twitterbot","GPTBot"]}
{"domain":"bootstrapcdn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hh.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["Yandex","YandexDirect","Googlebot","AdsBot-Google","AdsBot-Google-Mobile","LCC","ltx71 - (http://ltx71.com/)","bingbot","GPTBot","Claude-Web","ClaudeBot","ChatGPT-User","Google-Extended","FacebookBot","OAI-SearchBot","OAI-Fetcher","Anthropic-ai","CCBot","PerplexityBot","*"]}
{"domain":"mashable.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":108,"stanza_tokens":["AddSearchBot","AI2Bot","Ai2Bot-DeepResearchEval","Ai2Bot-Dolma","AmazonBuyForMe","Amazonbot","Amzn-SearchBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot","Applebot-extended","atlassian-bot","AutoRAG","AwarioSmartBot","AzureAI-SearchBot","Big Sur AI","bigsur.ai","Brandwatch","Bravebot","Brightbot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","Datenbank-Crawler","DeepSeekBot","Devin","Diffbot","Direqt Anomura","DuckAssistBot","Echobot Bot","ExaBot","FacebookBot","Factset_spyderbot","FirecrawlAgent","GPTBot","iAsk","iAskBot","iaskspider","ICC Crawler","ICC-Crawler","ImageSiftBot","imageSpider","Kangaroo Bot","Kangaroo-Bot","KlaviyoAIBot","Kunato","laion-huggingface-processor","LCC","LinerBot","LinkupBot","LINER Bot","Manus-User","meta-externalagent","meta-externalfetcher","meta-webindexer","mistral.ai","MistralAI-User","netEstate Imprint Crawler","Novellum AI Crawl","Novellum","NovaAct","OAI-SearchBot","omgili","opencode","PanguBot","Perplexity-User","PerplexityBot","PeopleInc-DCipher-Scraper/1.1.0","PetalBot","Poggio-Citations","QualifiedBot","SeekrBot","SemrushBot-OCOB","SemrushBotSwa","ShapBot","SBIntuitionsBot","Spider","TerraCotta","Timpibot","TavilyBot","Trae","TwinAgent","Velen Crawler","VelenPublicWebCrawler","Webzio-Extended","Wrtn","YouBot"]}
{"domain":"zenlogic.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ENOTFOUND"]}
{"domain":"ayhal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"www.gob.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot","googlebot-image","googlebot-mobile","AdsBot-Google"]}
{"domain":"npci.org.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News","Google-InspectionTool","GoogleOther","GoogleOther-Image","GoogleOther-Video","AdsBot-Google","AdsBot-Google-Mobile","Mediapartners-Google","APIs-Google","Google-CloudVertexBot","Google-Extended","Google-read-aloud","Bingbot","MSNBot","MSNBot-Media","GPTBot","ChatGPT-User","OAI-SearchBot","PerplexityBot","ClaudeBot","Claude-Web","anthropic-ai","Applebot","Applebot-Extended","FacebookBot","meta-externalagent","CCBot","ia_archiver","archive.org_bot","ia_archiver-web.archive.org","*"]}
{"domain":"truste.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adobelogin.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"suruga-ya.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"connatix.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"appier.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dian.gov.co","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"office365.us","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"strava.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":5,"stanza_tokens":["ClaudeBot","Google-Extended","GPTBot","Meta-ExternalAgent","*"]}
{"domain":"politico.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"hulu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":7,"stanza_tokens":["blinkx","blinkx_ff_spider","bingbot","Googlebot","Google-Extended","GPTBot","*"]}
{"domain":"mercadolibre.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User"],"open":[],"path_restricted":["Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["Amazonbot","PerplexityBot","Perplexity-User","ClaudeBot","Claude-User","GPTBot","ChatGPT-User","AdsBot-Google","AdsBot-Google-Mobile","*"]}
{"domain":"colliderporn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":3,"stanza_tokens":["GPTBot","ClaudeBot","*"]}
{"domain":"iubenda.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","Googlebot-Image","Mediapartners-Google*","GPTBot","ChatGPT-User","OAI-SearchBot","Google-Extended","Applebot-Extended","anthropic-ai","Amazonbot","ClaudeBot","Claude-Web","Cohere-ai","PerplexityBot","CCBot","Omgilibot","Omgili","Webzio-Extended","FacebookBot","Bytespider"]}
{"domain":"namasha.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","Slurp","msnbot","YandexBot","SeznamBot","discobot","dotbot","yacybot"]}
{"domain":"ohthree.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rings.solutions","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"upwork.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"evernote.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","bingbot"]}
{"domain":"max.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ad-delivery.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"ninisite.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hsforms.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"akamai.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"doujindesu.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"orcid.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"manhwaread.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"badoo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["MJ12bot","Googlebot","Mediapartners-Google","AdsBot-Google","AdsBot-Google-Mobile","msnbot","Yandex","*"]}
{"domain":"loopme.me","fetch_ok":false,"usable":false,"reason":"other","status":204,"error":null,"attempts":[204]}
{"domain":"case-battle.red","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"ezvizru.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"komikindo.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"easy4ipcloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"tripod.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"expedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","google-hoteladsverifier","Mediapartners-Google","AdsBot-Google-Mobile","AdsBot-Google","AdIdxBot","OAI-AdsBot","SemrushBot","OAI-SearchBot","ChatGPT-User","PerplexityBot","Perplexity-User","Claude-User","Claude-SearchBot"]}
{"domain":"mitre.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"one.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"meteociel.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","anthropic-ai"],"path_restricted":["Claude-SearchBot"]}],"stanza_token_count":21,"stanza_tokens":["Googlebot","AhrefsBot","AspiegelBot","BLEXBot","SemrushBot","Bytespider","ClaudeBot","Baiduspider-render","Sogou web spider","Sogou spider","Sogou inst spider","Sogou","newsai","Amazonbot","Applebot","Claude-SearchBot","AwarioSmartBot","AwarioRssBot","Googlebot-Image","bingbot","GoogleOther"]}
{"domain":"men.gov.ma","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xahani.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"hwcdn.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"desiviral.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"indoav.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"pitc.com.pk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oath.cloud","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"33apk.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"vu.edu.pk","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_VERIFY_LEAF_SIGNATURE","attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE","TimeoutError"]}
{"domain":"telegra.ph","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"savvis.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","CERT_HAS_EXPIRED"]}
{"domain":"wa.link","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"microsoftpersonalcontent.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mp4moviez.care","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"app.link","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"freebsd.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","GPTBot","CCBot","SemrushBot","MJ12bot","BLEXBot","dotbot","SeekportBot","AhrefsBot"]}
{"domain":"mangaraw.ac","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cdek.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":418,"error":null,"attempts":[418,418]}
{"domain":"mongodb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["mauibot","semrushbot","ahrefsbot","blexbot","seo spider","bingbot","*"]}
{"domain":"ign.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"google.co.id","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"sexvid1.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["BingBot","*"]}
{"domain":"umeng.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":26,"stanza_tokens":["*","Googlebot","Googlebot-Image","Googlebot-Video","Bingbot","msnbot","PermaBot","GPTBot","ChatGPT-User","OAI-SearchBot","Google-Extended","ClaudeBot","anthropic-ai","PerplexityBot","Bytespider","Baiduspider","Baiduspider-image","Sogou web spider","Sogou Pic Spider","360Spider","Applebot","DuckDuckBot","YandexBot","facebookexternalhit","Twitterbot","LinkedInBot"]}
{"domain":"minhngoc.net.vn","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"git-scm.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"ganamosnet.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"javwow.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"xvideo.vlog.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"v9r2yhjg.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"topgirl.co","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"education.dz","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"archive.ph","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"otm-r.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"dnse3.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ssrn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":4,"stanza_tokens":["GPTBot","ChatGPT-User","Google-Extended","*"]}
{"domain":"iqiyi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":14,"stanza_tokens":["openai","gptbot","chatgpt-user","OAl-SearchBot","claudebot","pangubot","petalbot","ChatGLM-Spider","ChatGLMBot","skyworkspider","perplexitybot","omigili","cohere-ai","*"]}
{"domain":"science.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":8,"stanza_tokens":["*","Google-Extended","anthropic-ai","Twitterbot","GPTBot","ChatGPT-User","CCBot","Applebot-Extended"]}
{"domain":"verify.hn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"test-uz.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":26,"stanza_tokens":["*","TurnitinBot","barkrowler","ClaudeBot","BLEXBot","MJ12bot","dotbot","SeekportBot","AhrefsBot","Amazonbot","serpstatbot","GPTBot","AwarioRssBot","AwarioSmartBot","CCBot","ChatGPT-User","anthropic-ai","Omgilibot","Omgili","FacebookBot","Diffbot","Bytespider","ImagesiftBot","cohere-ai","Claude-User","Claude-SearchBot"]}
{"domain":"orange.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"m.me","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"rfc-editor.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"intentiq.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hover.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["twitterbot","*"]}
{"domain":"futbol-libre.su","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"betweendigital.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"im-apps.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"mossru.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gama.ir","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"isappcloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"people.com.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"norton.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","GPTBot","Perplexity-User","PerplexityBot","Claude-User","ClaudeBot","GoogleOther","Google-Extended"]}
{"domain":"giex.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"marketingcloudapis.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ak-is2.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"opera.software","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"timeanddate.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"themoviedb.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":93,"stanza_tokens":["AddSearchBot","AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","Andibot","anthropic-ai","Applebot","Applebot-Extended","Awario","bedrockbot","bigsur.ai","Brightbot 1.0","Bytespider","CCBot","ChatGPT Agent","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Datenbank Crawler","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","FacebookBot","facebookexternalhit","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","LinerBot","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NovaAct","OAI-SearchBot","omgili","omgilibot","OpenAI","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","Thinkbot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","wpbot","YaK","YandexAdditional","YandexAdditionalBot","YouBot"]}
{"domain":"mradx.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ed.gov","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"gamisco.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"uisp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"euronews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":12,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","PerplexityBot","FirecrawlAgent","AndiBot","ExaBot","PhindBot","YouBot","GPTBot","CCBot","Google-Extended"]}
{"domain":"xn--12c1ezaww.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"news.com.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":65,"stanza_tokens":["ScreamingFrogSEOSpider","proximic","Outbrain","Yahoo Ad monitoring","SemrushBot","facebookexternalhit","ias_crawler","identrics","MeltwaterNews","links.streem.com.au","LinkedInBot","redditbot","Mediapartners-Google","Google-InspectionTool","Adsbot-Google","Storebot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","APIs-Google","Google-Ads-Creatives-Assistant","Google-adwords","Google-amphtml","Google-cloudvertexbot","Google-play-newsstand","Google-read-aloud","Google-Safety","Google-site-verification","Google-Structured-Data-Testing-Tool","Googleimageproxy","Applebot","AppleNewsBot","adidxbot","admantx","admantx-usbatch/3.1","AmazonAdBot","Chrome-Lighthouse","Verity","Leikibot","parselybot","peer39_crawler/1.0","pingdom","Pinterestbot","PubMatic Crawler Bot","Mobian","Slack-ImgProxy","Slackbot","Slackbot-LinkExpanding","thetradedesk","Uptimebot","yahoo-linkpreview","Googlebot","Googlebot-Image","Googlebot-News","Googlebot-Video","GoogleOther","GoogleOther-Image","GoogleOther-Video","Bingbot","bingpreview","MicrosoftPreview","DuckDuckBot","Slurp","yahoo","Google-Extended","*"]}
{"domain":"app-ads-services.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"beget.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"yimg.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"moviespage.xyz","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"thesun.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":50,"stanza_tokens":["*","Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News","Google-InspectionTool","GoogleOther","APIs-Google","Mediapartners-Google","DuckDuckBot","Slurp","Twitterbot","Bingbot","adidxbot","MicrosoftPreview","Applebot","AdsBot-Google","AmazonAdBot","facebookexternalhit","Yahoo Ad Monitoring","parse.ly scraper","Screaming Frog SEO Spider","SemrushBot","Botify","OnCrawl","Chrome-Lighthouse","Slackbot-LinkExpanding","Slack-ImgProxy","Slackbot","Snap URL Preview Service","proximic","GumGum-Bot","TTD-Content","Pubmatic","ias_crawler","outbrain","Leikibot","weborama-fetcher","AHC/2.1","SirdataBot","peer39_crawler/1.0","EchoboxBot/1.0","CriteoBot/0.1","AdDefend Site Context Crawler/1.0","SmartologyBot","Dianomi","ChatGPT-User","GPTBot","OAI-SearchBot","snews"]}
{"domain":"squareup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vidzp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"segment.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Swiftbot"]}
{"domain":"123tt.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yasyadong01.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kgslb.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"unicef.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"99rdd.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"vtb.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"logitech.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lnkd.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"digitallocker.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bytedns3.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"amp-endpoint2.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nba.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vidvf.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"clevertap-prod.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aniview.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"samsungdm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dpbossss.boston","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"filmyfly.cafe","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"xxxbp.pro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["ia_archiver","*"]}
{"domain":"bitbucket.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","008"]}
{"domain":"isaidub.guru","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"mirror.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":[],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":22,"stanza_tokens":["*","daumoa","Sosospider","rogerbot","semetrical","Googlebot-News","AhrefsBot","GPTBot","OAI-SearchBot","Applebot-Extended","PerplexityBot","Perplexity-ai","CCBot","YouBot","ClaudeBot","anthropic-ai","Claude-Web","grapeshot","bingbot","AmazonAdBot","ozone","Meta-ExternalAgent"]}
{"domain":"wunderground.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AmazonAdBot","*"]}
{"domain":"bls.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["archive.org_bot","*"]}
{"domain":"ydparty06.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"xnhau.shop","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"wechat.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"6qdgn5ym.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"magonet.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"hdhub4u.limo","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"xn--2-wxfc3gwbi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"moviezwap.love","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"eskimi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"asu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"justice.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","usasearch"]}
{"domain":"ads-twitter.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"usgovtrafficmanager.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mp4moviez.fan","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"kamababax.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"filmyzilla39.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"mp4moviez.bot","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"hhs.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"house.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"moviesda30.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"schwab.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"videyly.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"springserve.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"iponweb.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"videlly.space","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"olxide.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"xnhau.cx","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"ntp.br","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"mingky13.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"xnhau.bet","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"filmyzilla37.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"acronis.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":12,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","Twitterbot","dotbot"]}
{"domain":"5movierulz.soccer","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"5movierulz.church","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"thscore99.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"5movierulz.forsale","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"sonobi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dnsimple.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ufl.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aibixby.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mosspf.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pwc.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"filmyfly.camp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"redd.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"5movierulz.camera","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"mamtpo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ahrefs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aboutcookies.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"discomax.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fyber.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"sbxh1.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"engadget.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rspamd.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Googlebot","Bingbot","Twitterbot","facebookexternalhit","*"]}
{"domain":"metamask.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"barnesandnoble.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","adsbot-google","AccessibleWebBot","Nutch","AhrefsBot","AhrefsSiteAudit","MJ12bot","Pinterest"]}
{"domain":"blog.google","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"1688.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Baiduspider","Googlebot","bingbot","MSNBot","360Spider"]}
{"domain":"eyeota.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"mindbox.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":39,"stanza_tokens":["facebookexternalhit","*","Yandex","Googlebot","Googlebot-Image","Googlebot-News","Googlebot-Video","Google-Extended","Googlebot/2.1","bingbot","Bingbot/2.0","ChatGPT-User","GPTBot","GPTBot/1.1","GPTBot/1.2","OAI-SearchBot","Meta-ExternalFetcher","Meta-ExternalAgent","FacebookBot","ClaudeBot","Claude-Web","anthropic-ai","Claude-User","Claude-SearchBot","Applebot","Applebot-Extended","Amazonbot","NovaAct","Bytespider","CCBot","Perplexity-User","PerplexityBot","DuckAssistBot","cohere-ai","cohere-training-data-crawler","AI2Bot","YouBot","MistralAI-User","Kangaroo Bot"]}
{"domain":"dbankcdn.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"ctadns.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dropboxusercontent.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"privacymanager.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cootlogix.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"apple.news","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"samsungnyc.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tribalfusion.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["GoogleOther"]}
{"domain":"medicalnewstoday.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":29,"stanza_tokens":["Mediapartners-Google*","Nutch","Slurp","AmazonAdbot","Amazonbot","Amzn-SearchBot","Amzn-User","amazon-kendra","amazon-QBusiness","bedrockbot","aws-quick-on-behalf-of-","AmazonProductDiscoverybot","AmazonBuyForMe","AmazonSellerInitiatedListing","NovaAct","*","GPTBot","Applebot-Extended","anthropic-ai","Bytespider","CCBot","ClaudeBot","Diffbot","FacebookBot","Meta-ExternalAgent","omgili","Timpibot","Webzio-Extended","DDM-DCipher/1.0.7"]}
{"domain":"bigcartel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"proofpoint.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","meta-externalagent","Meta-ExternalAgent","AI2Bot","Diffbot","Omgili","Webzio-Extended"]}
{"domain":"adkernel.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"yotpo.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"truecaller.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"primis.tech","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"amocrm.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","GoogleBot","*"]}
{"domain":"gvt3.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"av380.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"itu.int","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"traffmonetizer.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"variety.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":26,"stanza_tokens":["*","anthropic-ai","cohere-ai","omgili","omgilibot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ChatGPT","DeepSeek","DeepSeekBot","GPTBot","OAI-SearchBot","ClaudeBot","Claude-Web","Diffbot","FacebookBot","Google-Extended","Meta-ExternalAgent","Meta-ExternalFetcher","PerplexityBot","Timpibot","Webzio-Extended","YouBot","cXensebot"]}
{"domain":"mediago.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"virtualearth.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"semrush.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"threema.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mgid.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["*","Google-Extended","GPTBot","ChatGPT-User","CCBot","PerplexityBot","anthropic-ai","Claude-Web","ClaudeBot","Amazonbot","Omgilibot","FacebookBot","Applebot","Bytespider","Diffbot","ImagesiftBot","Omgili","YouBot"]}
{"domain":"vox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":["GPTBot"],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":105,"stanza_tokens":["Amazonbot","Applebot","GPTBot","A6-Indexer","AhrefsBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot-Extended","Baiduspider","betaBot","BLEXBot","botify","brands-bot-logo","Brightbot","Bytespider","calculon","CCBot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","collection@infegy.com","Cotoyogi","Crawl4AI","CrunchBot","deepcrawl","DeepSeekBot","Diffbot","DirBuster","DTS","DuckAssistBot","EasouSpider","FacebookBot","FirecrawlAgent","FriendlyCrawler","GlutenCrawler","Google-CloudVertexBot","Google-Extended","GrapeshotCrawler","iaskspider","ICC-Crawler","ImagesiftBot","img2dataset","JackCrawl","Jetslide","linkapedia_bot","Mediatoolkitbot","MegaIndex","meta-externalagent","meta-externalfetcher","Microsoft BITS","Microsoft URL Control","Microsoft-WebDAV-MiniRedir","MistralAI-User","news-please","Novellum","NTENTbot","Nutch","omgili","Omgilibot","OpenLinkProfiler","Perplexity-User","PerplexityBot","PetalBot","Primalbot","python-requests","Re-Animator","Riddler","SafeDNSBot","SBIntuitionsBot","Scrapy","SearchAtlas","SemanticScholarBot","SemrushBot","SentiBot","Sogou","Spider-Pig","sqlmap","StorygizeBot","Test Certificate Info","TikTokSpider","Timpibot","Turnitin","uipbot","vBSEO","VegeBot","VelenPublicWebCrawler","Web Link Validator","WebCruiser"]}
{"domain":"charlestonroadregistry.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"openssl.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"amazonaws.com.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"photobucket.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","SemrushBot","SemrushBot-SA","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SemrushBot-CT","SemrushBot-BM","SemrushBot-SEOAB","Bytespider","PetalBot"]}
{"domain":"cpmstar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"curseforge.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"dropbox-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"americanexpress.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AhrefsBot"]}
{"domain":"like.video","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"myvzw.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"xincache.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mwbsys.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"here.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ihg.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"revenuecat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sec.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mastercard.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sony.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"glassdoor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":25,"stanza_tokens":["*","ia_archiver","OmniExplorer_Bot","Mediapartners-Google","Baiduspider","GPTBot","Google-Extended","Amazonbot","anthropic-ai","GoogleOther","Claude-Web","Perplexity","Cohere","cohere-ai","Applebot-Extended","Google-CloudVertexBot","CCBot","Omgilibot","Omgili","FacebookBot","Bytespider","Diffbot","Youbot","FriendlyCrawler","img2dataset"]}
{"domain":"gshifen.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"geoedge.be","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bytedance.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"home.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":53,"stanza_tokens":["*","Googlebot","Googlebot-Image","Googlebot-News","Googlebot-Video","GoogleOther","Google-InspectionTool","Storebot-Google","Bingbot","360Spider","Baiduspider","coccocbot","Daum","DuckDuckBot","Ecosia_bot","MojeekBot","Yeti","SeznamBot","Sogou web spider","Yahoo! Slurp","YandexAccessibilityBot","YandexMobileBot","Yandex","Amazonbot","ClaudeBot","Claude-User","Claude-SearchBot","CCBot","Google-Extended","FacebookBot","MistralAI-User","GPTBot","ChatGPT-User","OAI-SearchBot","PerplexityBot","Perplexity-User","archive.org_bot","AdsBot-Google","AdsBot-Google-Mobile","Chrome-Lighthouse","facebookexternalhit","Google-Read-Aloud","Google-Safety","Hatena","linkedinbot","Mediapartners-Google","Pinterestbot","SchemaBot","SiteAuditBot","SiteimproveBot","Slackbot-LinkExpanding","Telegram","Twitterbot"]}
{"domain":"adsco.re","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot","*"]}
{"domain":"mintegral.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"smi2.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"datto.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"medu.ir","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bitrixphone.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"steamcontent.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"e-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kde.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"e-volution.ai","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"lync.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"sc-static.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"simpli.fi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adobess.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cloudfunctions.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"chartbeat.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ENOTFOUND"]}
{"domain":"icann.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"clients-cdnnow.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"xserver.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_UNSUPPORTED_PROTOCOL","attempts":["ERR_SSL_UNSUPPORTED_PROTOCOL","ERR_SSL_UNSUPPORTED_PROTOCOL"]}
{"domain":"mixcloud.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"delegaciondelgobierno.gob.es","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"verizon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wswebpic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"script.ac","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"systemdns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"windy.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"amp-endpoint3.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sharethis.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"typekit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","SemrushBot","SiteAuditBot"]}
{"domain":"census.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","W3C-checklink","Googlebot","Yahoo! Slurp","bingbot","usasearch"]}
{"domain":"microsoft365.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"innovid.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AhrefsSiteAudit","AhrefsBot"]}
{"domain":"xundns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"nipr.mil","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"igodigital.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tiqcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"liftoff-creatives.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"fliphtml5.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":146,"stanza_tokens":["Baiduspider","baiduspider","Baiduspider-image","Baiduspider-video","Baiduspider-news","Baiduspider-favo","Baiduspider-cpro","Baiduspider-ads","Sosospider","sogou spider","Sogou News Spider","Sogou inst spider","Sogou spider2","Sogou blog","Sogou Orion spider","Sogou web spider","YodaoBot","Jike Spider","360Spider","360spider-image","360spider-video","HaosouSpider","YisouSpider","EasouSpider","ToutiaoSpider","Yahoo! Slurp China","EtaoSpider","yaanibot","meta-externalagent","Meta-ExternalAgent","FacebookBot","Amazonbot","Bytespider","PetalBot","PanguBot","ChatGLM-Spider","CCBot","cohere-ai","cohere-training-data-crawler","Diffbot","AI2Bot","AI2Bot-Dolma","ImagesiftBot","img2dataset","omgili","Omgilibot","Webzio-Extended","YouBot","Timpibot","Kangaroo Bot","Sidetrade indexer bot","VelenPublicWebCrawler","iaskspider","iaskspider/2.0","360AISpider","TouTiaoSpider","MoonshotAI","Kimi-Spider","DeepSeekBot","DeepSeek-Crawler","TencentBot","BaichuanBot","BaichuanAI","SparkBot","xfspider","SkyworkBot","MiniMax-Spider","QwenSpider","DashScope","WenxinSpider","ERNIE-Bot","SenseBot","SenseChat","YuewenAI","YiBot","01ai","SemrushBot","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SemrushBot-CT","SemrushBot-OCOB","SemrushBot-FT","SplitSignalBot","AhrefsBot","MJ12bot","DotBot","Rogerbot","BLEXBot","DataForSeoBot","Screaming Frog SEO Spider","SearchmetricsBot","SeobilityBot","SeekportBot","SeekrBot","SEOkicks-Robot","SEOkicks","SEOstats","SeoSiteCheckup","SEOdiver Bot"]}
{"domain":"polymarket.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mparticle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":41,"stanza_tokens":["*","YouBot","Webzio-Extended","VelenPublicWebCrawler","Timpibot","Sidetrade indexer bot","Scrapy","PetalBot","PerplexityBot","omgilibot","omgili","OAI-SearchBot","Meta-ExternalFetcher","Meta-ExternalAgent","Kangaroo Bot","ISSCyberRiskCrawler","img2dataset","ImagesiftBot","ICC-Crawler","iaskspider/2.0","GPTBot","GoogleOther-Video","GoogleOther-Image","GoogleOther","Google-Extended","FriendlyCrawler","facebookexternalhit","FacebookBot","Diffbot","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","cohere-ai","ClaudeBot"]}
{"domain":"purdue.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"example.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ninjarmm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","PerplexityBot","ClaudeBot","Claude-SearchBot","Google-Extended"]}
{"domain":"dondominio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ad.gt","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","GoogleOther"]}
{"domain":"feedburner.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"uiuc.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ria.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"unruly.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"duke.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gos-gsp.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ltwebstatic.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"symantec.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"nodejs.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"via.google","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"nic.xyz","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"quantummetric.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"shorturl.at","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"bazaarvoice.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["heritrix","*"]}
{"domain":"trustedstack.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"comodoca.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"laravel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"axon.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mmcdn.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"earthlink.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"techradar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","AmazonAdBot","bytespider","mistralai","cohere","ai2bot","youbot","omgili","diffbot","kangaroo","img2dataset","amazonbot","amazon-qbusiness","meta-externalagent","meta-webindexer"]}
{"domain":"dynamics.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"istockphoto.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"inc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":27,"stanza_tokens":["*","Googlebot","msnbot","bingbot","Twitterbot","Mediapartners-Google","Google-Extended","ScoutJet","Yandex","baiduspider","DuckDuckBot","BomboraBot","grapeshot","ccbot","GPTBot","Bard","Jasper","YouChat","Perplexity","Claude","Claude-Web","Neeva","Copy.ai","Writesonic","Anyword","AI21","meta-externalagent"]}
{"domain":"juicyscore.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"ay.delivery","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"lin.ee","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["genieo"]}
{"domain":"algolia.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"nyu.edu","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"cookiebot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gallup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["amazon-kendra","GPTBot","ChatGPT-User","PerplexityBot","Perplexity-User","ClaudeBot","Claude-User","Claude-SearchBot","OAI-SearchBot","Google-Extended","MistralAI-User","cohere-ai","*","PetalBot"]}
{"domain":"wps.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"smailru.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"doppiocdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"a-mx.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sendbird.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":14,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","PerplexityBot","FirecrawlAgent","AndiBot","ExaBot","PhindBot","YouBot","GPTBot","CCBot","Google-Extended","Googlebot","Bingbot"]}
{"domain":"xboxservices.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vercel-dns.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"useinsider.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AdsBot-Google","AdsBot-Google-Mobile"]}
{"domain":"register.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"cctv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unc.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"asana.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"didiglobal.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"msappproxy.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","MS Search 6.0 Robot"]}
{"domain":"miro.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Twitterbot","facebookexternalhit","Slackbot-LinkExpanding"]}
{"domain":"semasio.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"digicertdns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"constellix.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jimdofree.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"peacocktv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","msnbot","MauiBot (crawler.feedback+wc@gmail.com)"]}
{"domain":"wpengine.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"garenanow.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"congress.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":129,"stanza_tokens":["*","AddSearchBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","amazon-kendra","Amazonbot","AmazonBuyForMe","Andibot","Anomura","anthropic-ai","atlassian-bot","Awario","bedrockbot","bigsur.ai","Bravebot","Brightbot 1.0","BuddyBot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT Agent","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","Cloudflare-AutoRAG","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Crawlspace","Datenbank Crawler","DeepSeekBot","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","FacebookBot","facebookexternalhit","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","Google-Firebase","Google-NotebookLM","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iAskBot","iaskspider","iaskspider/2.0","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","LAIONDownloader","LCC","LinerBot","Linguee Bot","LinkupBot","Manus-User","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","meta-webindexer","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NotebookLM","NovaAct","OAI-SearchBot","omgili","omgilibot","OpenAI","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot"]}
{"domain":"semanticscholar.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"npmjs.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"slate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":302,"stanza_tokens":["*","AmazonAdBot","Amazon-Advertising-ad-standards-bot","008","AdsTxtCrawler","Ai2Bot-Dolma","Amazonbot","Applebot-Extended","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","CloudVertexBot","cohere-training-data-crawler","Cotoyogi","Dataprovider.com","Datenbank Crawler","dcrawl","FacebookBot","Google-Extended","GoogleOther","GPTBot","HelloworkJobPostingBot","HTTrack","HTTrack 3.0","ICC-Crawler","imageSpider","IndeedJobBot","Kangaroo Bot","KrawlerBot","laion-huggingface-processor","LCC","meta-externalagent","MetaInspector","MetaJobBot","netEstate Imprint Crawler","newspaper","Nutch","nyt_scraping","Offline Explorer","omgili","OpenindexSpider","PanguBot","Potions","SBIntuitionsBot","Scrapy","ServerHunterSpider","Spider","StatsDroneBot","Timpibot","VelenPublicWebCrawler","webzio-extended","A360-Search","aa","ActiveComply","Adagiobot","adbeat_bot","adidxbot","AdkernelTopicCrawler","admantx","AdsBot-Google","AdsBot-Google-Mobile","Ahrefs Site Audit","AhrefsBot","AhrefsSiteAudit","aiHitBot","Amazing-SearchBot","AmazingTest-SearchBot","Amazon AdBot","AmazonProductDiscovery","AndersPinkBot","ArchiveBot","Assaybot","Attracta","AudigentAdBot","Audisto Crawler","Automattic Analytics Crawler","Awario","AwarioBot","AwarioSmartBot","Barkrowler","BigUpDataBot","Bing Ads","BitSightBot","Blackboard","BLEXBot","BomboraBot","botify","BrandVerity","BrightEdge Bot","BrightEdge Crawler","Buck","CaliberBot","Caliperbot","channable","CheckMarkNetwork","Cincraw","ClarityBot","Clearscopebot","Clickagy Intelligence Bot"]}
{"domain":"fullstory.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"umd.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":4,"stanza_tokens":["*","GPTBot","Google-Extended","PerplexityBot"]}
{"domain":"lego.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"inbake.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ENOTFOUND"]}
{"domain":"jwplayer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"posthog.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot","Bingbot","DuckDuckBot","Yeti"]}
{"domain":"secu100.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ECONNREFUSED",404]}
{"domain":"made-in-china.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["*","AdsBot-Google","AdsBot-Google-Mobile","SemrushBot","Baiduspider","GPTBot","OAI-SearchBot","ChatGPT-User","Google-Extended","Gemini-Deep-Research","ClaudeBot","Claude-SearchBot","Claude-User","GrokBot","PerplexityBot","Perplexity-User","Applebot-Extended","CCBot"]}
{"domain":"hotmail.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"gettyimages.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"bixbyllm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hbomax.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"generatepress.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Googlebot","Bingbot","*"]}
{"domain":"buzzoola.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oculus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot","*"]}
{"domain":"customer.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Googlebot","bingbot","*"]}
{"domain":"tnresults.nic.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"adobestats.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"trs-dns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"onmicrosoft.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"utoronto.ca","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"messenger.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":17,"stanza_tokens":["Amazonbot","Applebot-Extended","Brightbot","ClaudeBot","Google-Extended","GPTBot","PerplexityBot","PetalBot","Scrapy","uptimerobot","viberbot","YaK","Yandex","Yeti","facebookexternalhit","meta-externalads","*"]}
{"domain":"onrender.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"tenor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"guardian.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":39,"stanza_tokens":["*","Mediapartners-Google","NewsNow","CCBot","TurnitinBot","PetalBot","MoodleBot","FacebookBot","Bytespider","Mojeek","JenkersBot","Seekr","YouBot","Arquivo-web-crawler","coccocbot-web","SeznamBot","PerplexityBot","yacy","anthropic-ai","ClaudeBot","Claude-SearchBot","Claude-User","AwarioRssBot","AwarioSmartBot","SentiOne","ImageSift","Applebot-Extended","YandexAdditional","YandexAdditionalBot","scalepostAI","Buck","meta-externalagent","Amazonbot","amazon-QBusiness","DuckAssistBot","Google-CloudVertexBot","Amzn-SearchBot","AhrefsBot","AhrefsSiteAudit"]}
{"domain":"bumble.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rollingstone.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":26,"stanza_tokens":["*","anthropic-ai","cohere-ai","omgili","omgilibot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ChatGPT","DeepSeek","DeepSeekBot","GPTBot","OAI-SearchBot","ClaudeBot","Claude-Web","Diffbot","FacebookBot","Google-Extended","Meta-ExternalAgent","Meta-ExternalFetcher","PerplexityBot","Timpibot","Webzio-Extended","YouBot","cXensebot"]}
{"domain":"lfstrm.tv","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"freefiremobile.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"apachefriends.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ucdavis.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"www.gov.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hetzner.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"altervista.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"nest.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"so.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"beget.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"craigslist.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cloudflarestorage.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"opera.technology","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","UND_ERR_CONNECT_TIMEOUT"]}
{"domain":"paywb.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"onetrust.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"usc.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"grafana.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"axios.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["Twitterbot/*","*","CCBot","Bytespider","Diffbot","FacebookBot","ImagesiftBot","Scrapy","Amazonbot"]}
{"domain":"co.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"immunet.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"gmoregistry.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_UNRECOGNIZED_NAME","attempts":["ERR_SSL_TLSV1_UNRECOGNIZED_NAME","ERR_SSL_TLSV1_UNRECOGNIZED_NAME"]}
{"domain":"mongodb.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"express.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":[],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":25,"stanza_tokens":["*","grapeshot","Googlebot-News","ia_archiver","Nutch","ias_crawler","ias_wombles","ias-ie","daumoa","Sosospider","rogerbot","semetrical","AhrefsBot","GPTBot","OAI-SearchBot","Applebot-Extended","PerplexityBot","Perplexity-ai","CCBot","YouBot","ClaudeBot","anthropic-ai","Claude-Web","AmazonAdBot","ozone"]}
{"domain":"alikunlun.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"quantcount.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Googlebot","Googlebot-Image","AdsBot-Google","GoogleOther","*"]}
{"domain":"pcmag.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":108,"stanza_tokens":["AddSearchBot","AI2Bot","Ai2Bot-DeepResearchEval","Ai2Bot-Dolma","AmazonBuyForMe","Amazonbot","Amzn-SearchBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot","Applebot-extended","atlassian-bot","AutoRAG","AwarioSmartBot","AzureAI-SearchBot","Big Sur AI","bigsur.ai","Brandwatch","Bravebot","Brightbot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","Datenbank-Crawler","DeepSeekBot","Devin","Diffbot","Direqt Anomura","DuckAssistBot","Echobot Bot","ExaBot","FacebookBot","Factset_spyderbot","FirecrawlAgent","GPTBot","iAsk","iAskBot","iaskspider","ICC Crawler","ICC-Crawler","ImageSiftBot","imageSpider","Kangaroo Bot","Kangaroo-Bot","KlaviyoAIBot","Kunato","laion-huggingface-processor","LCC","LinerBot","LinkupBot","LINER Bot","Manus-User","meta-externalagent","meta-externalfetcher","meta-webindexer","mistral.ai","MistralAI-User","netEstate Imprint Crawler","Novellum AI Crawl","Novellum","NovaAct","OAI-SearchBot","omgili","opencode","PanguBot","Perplexity-User","PerplexityBot","PeopleInc-DCipher-Scraper/1.1.0","PetalBot","Poggio-Citations","QualifiedBot","SeekrBot","SemrushBot-OCOB","SemrushBotSwa","ShapBot","SBIntuitionsBot","Spider","TerraCotta","Timpibot","TavilyBot","Trae","TwinAgent","Velen Crawler","VelenPublicWebCrawler","Webzio-Extended","Wrtn","YouBot"]}
{"domain":"samsungelectronics.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"lww.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"entrepreneur.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dnsv5.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"aws-prd.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"trs-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"klarna.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"abcnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":12,"stanza_tokens":["claritybot","GPTBot","Google-Extended","CCBot","ChatGPT-User","anthropic-ai","Omgilibot","Omgili","FacebookBot","Bytespider","*","Applebot"]}
{"domain":"gameanalytics.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"samba.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bplslb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"senate.gov","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"msu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"smi2.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":32,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","ChatGPT-User/2.0","GPTBot","anthropic-ai","ClaudeBot","claude-web","PerplexityBot","Perplexity-User","Google-Extended","Amazonbot","Applebot","Applebot-Extended","FacebookBot","meta-externalagent","LinkedInBot","DuckAssistBot","Bytespider","cohere-ai","AI2Bot","CCBot","Diffbot","omgili","TimpiBot","YouBot","MistralAI-User","SBIntuitionsBot","GoogleOther","YandexBot","Googlebot","bingbot"]}
{"domain":"serving-sys.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.com.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"ingage.tech","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"digg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ticketmaster.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"energy.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adthrive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vkvideo.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"kinopoisk.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","*"]}
{"domain":"signal.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"hoyoverse.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"sogou.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["Sogou web spider","Sogou inst spider","Sogou spider2","Sogou blog","Sogou News Spider","Sogou Orion spider","JikeSpider","Sosospider","Googlebot","MSNBot","Baiduspider","360spider","Baiduspider-image","YoudaoBot","*"]}
{"domain":"rainberrytv.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"netscape.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"adswizz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tq-tungsten.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zdassets.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pacloudflare.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ubc.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"coinmarketcap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"tappx.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"broadcom.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"siriusxm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","sxm-gsa"]}
{"domain":"adsmoloco.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"centrastage.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nejm.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":15,"stanza_tokens":["*","Sosospider","AdsBot-Google","AdsBot-Google-Mobile","Googlebot","GPTBot","ChatGPT-User","CCBot","Google-Extended","PerplexityBot","SemanticScholarBot","AhrefsBot","BLEXBot","DataForSeoBot","SemrushBot"]}
{"domain":"nordvpn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["ia_archiver","archive.org_bot","ia_archiver-web.archive.org","*"]}
{"domain":"garena.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["facebookexternalhit","Facebot","meta-externalagent","Meta-ExternalFetcher","*"]}
{"domain":"last.fm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Bingbot","Yandexbot","*"]}
{"domain":"illinois.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hostpoint.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"armgs.team","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ENOTFOUND"]}
{"domain":"discordapp.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"web.dev","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"notion.site","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"thousandeyes.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"clickup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AdsBot-Google","AdsBot-Google-Mobile"]}
{"domain":"wayfair.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"disney.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"nielsen.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fanduel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"weibo.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"opensource.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bunny.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ey.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","Google-AIO","PerplexityBot","Perplexity-User","GPTBot","ChatGPT-User","Google-Extended","CCBot","anthropic-ai","BlueJBot","Spacecat/1.0"]}
{"domain":"web.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":21,"stanza_tokens":["*","googlebot-news","AI2Bot","Ai2Bot","Amazonbot","Applebot-Extended","CCBot","Cincraw","ClaudeBot","cohere-ai","Diffbot","FriendlyCrawler","GPTBot","ImagesiftBot","img2dataset","Meta-ExternalAgent","PetalBot","Semanticbot","TimpiBot","VelenPublicWebCrawler","Yandex"]}
{"domain":"ny.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sfgate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","anthropic-ai"],"open":[],"path_restricted":["Claude-SearchBot"]}],"stanza_token_count":23,"stanza_tokens":["ai2bot-dolma","anthropic-ai","applebot-extended","bytespider","ccbot","claude-user","claude-web","claudebot","cohere-ai","diffbot","duckassistbot","Friendlycrawler","omgili","omgilibot","perplexity-user","perplexitybot","scrapy","timpibot","youbot","webzio-extended","kangaroo-bot","AmazonAdBot","*"]}
{"domain":"infocyte.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"spot.im","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"livedns.co.uk","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"douban.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Wandoujia Spider","Mediapartners-Google"]}
{"domain":"vkuseraudio.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":418,"error":null,"attempts":[418,418]}
{"domain":"hatena.ne.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"mioffice.cn","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"smh.com.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":52,"stanza_tokens":["*","Google-CloudVertexBot","Google-Extended","ChatGPT-User","GPTBot","OAI-SearchBot","OAISearch","anthropic-ai","claude-web","claudebot","facebookbot","meta-externalagent","meta-externalfetcher","applebot-extended","perplexitybot","cohere-ai","youbot","amazonbot","aliyunsecbot","audigentadbot","awariorssbot","awariosmartbot","blexbot","bytespider","ccbot","dataforseobot","diffbot","duckassistbot","echoboxbot","friendlycrawler","ia_archiver","imagesiftbot","mycentralaiscraperbot","newsnow","news-please","omgili","omgilibot","webzio-extended","peer39_crawler","peer39_crawler/1.0","quillbot.com","quora-bot","scrapy","seekrbot","seznamhomepagecrawler","taragroup intelligent bot","timpibot","turnitinbot","viennatinybot","jetslide","magpie-crawler","poseidon research crawler"]}
{"domain":"about.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":112,"stanza_tokens":["*","Twitterbot","SemrushBot","SemrushBot-SA","asterias","BackDoorBot/1.0","Black Hole","BlowFish/1.0","BotALot","BuiltBotTough","Bullseye/1.0","BunnySlippers","Cegbfeieh","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","CopyRightCheck","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","DittoSpyder","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","ExtractorPro","Foobot","Harvest/1.5","hloader","httplib","humanlinks","InfoNaviRobot","JennyBot","Kenjin Spider","Keyword Density/0.9","LexiBot","libWeb/clsHTTP","LinkextractorPro","LinkScan/8.1a Unix","LinkWalker","LNSpiderguy","lwp-trivial","lwp-trivial/1.34","Mata Hari","Microsoft URL Control - 5.01.4511","Microsoft URL Control - 6.00.8169","MIIxpc","MIIxpc/4.2","Mister PiX","moget","moget/2.1","mozilla/4","Mozilla/4.0 (compatible; BullsEye; Windows 95)","Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)","Mozilla/4.0 (compatible; MSIE 4.0; Windows 98)","Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)","Mozilla/4.0 (compatible; MSIE 4.0; Windows XP)","Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000)","Mozilla/4.0 (compatible; MSIE 4.0; Windows ME)","mozilla/5","NetAnts","NICErsPRO","Offline Explorer","Openfind","Openfind data gathere","ProPowerBot/2.14","ProWebWalker","QueryN Metasearch","RepoMonkey","RepoMonkey Bait & Tackle/v1.01","RMA","SiteSnagger","SpankBot","spanner","suzuran","Szukacz/1.4","Teleport","TeleportPro","Telesoft","The Intraformant","TheNomad","TightTwatBot","Titan","toCrawl/UrlDispatcher","True_Robot","True_Robot/1.0","turingos","URLy Warning","VCI","VCI WebViewer VCI WebViewer Win32","Web Image Collector","WebAuto","WebBandit","WebBandit/3.50","WebCopier","WebEnhancer","WebmasterWorldForumBot","WebSauger","Website Quester"]}
{"domain":"weather.gov","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"scmp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["AmazonAdBot","*","NewsNow","grapeshot","Googlebot"]}
{"domain":"pruffme.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ioam.de","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"redditmedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","Googlebot","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","Googlebot-Image","Googlebot-Video","bingbot","MSNBot-Media","Applebot","ClarityBot","Neevabot"]}
{"domain":"imf.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"samsungads.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"sakura.ne.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"technorail.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","CERT_HAS_EXPIRED","TimeoutError"]}
{"domain":"imoim.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wac-dc-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sascdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TypeError","attempts":["TypeError","TimeoutError"]}
{"domain":"colorado.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nubank.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["DatoCmsSearchBot","*"]}
{"domain":"getpocket.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"umbrella.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"chartboost.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"zend.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unity.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Baiduspider","Sosospider","sogou spider","*"]}
{"domain":"privacy-center.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hpe.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"2gis.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nflxext.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"npo-iskra.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ccgateway.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"applvn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mlstatic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","EAI_AGAIN"]}
{"domain":"hihonor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"auvik.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","*"]}
{"domain":"geotrust.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"shodan.io","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"churnzero.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hyatt.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sjtu.edu.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"wtxcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"accenture.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oraclecloud.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hinet.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_VERIFY_LEAF_SIGNATURE","attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE","ENOTFOUND"]}
{"domain":"akstat.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"doordash.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"visualwebsiteoptimizer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["*","GPTBot","ChatGPT Agent","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","Claude-User","Claude-SearchBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GoogleAgent-Mariner","Google-CloudVertexBot","CloudVertexBot","Meta-ExternalAgent","Meta-ExternalFetcher","meta-externalagent","meta-externalfetcher","PerplexityBot","Perplexity-User","Applebot","Applebot-Extended","MistralAI-User","MistralAI-User/1.0","cohere-ai","cohere-training-data-crawler","YouBot","PhindBot","Amazonbot","Diffbot","Bytespider","CCBot"]}
{"domain":"kunlungr.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"vp3n.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"trs-dns.org","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dnsip.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wikia.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"locaweb.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["Googlebot"]}
{"domain":"geocomply.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","PerplexityBot","Google-Extended","CCBot","OAI-SearchBot"]}
{"domain":"mos.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"secb2b.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tospotify.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wetransfer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yammer.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"e1c-ops.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"qz.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"managed-vps.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"volces.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"splunkcloud.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_PACKET_LENGTH_TOO_LONG","attempts":["ERR_SSL_PACKET_LENGTH_TOO_LONG","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"audible.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["AdsBot-Google-Mobile","AdsBot-Google","*","GoogleOther"]}
{"domain":"xxpkg.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"DEPTH_ZERO_SELF_SIGNED_CERT","attempts":["DEPTH_ZERO_SELF_SIGNED_CERT","DEPTH_ZERO_SELF_SIGNED_CERT"]}
{"domain":"ogury.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"trs-dns.info","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"home-assistant.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gmx.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":21,"stanza_tokens":["*","googlebot-news","AI2Bot","Ai2Bot","Amazonbot","Applebot-Extended","CCBot","Cincraw","ClaudeBot","cohere-ai","Diffbot","FriendlyCrawler","GPTBot","ImagesiftBot","img2dataset","Meta-ExternalAgent","PetalBot","Semanticbot","TimpiBot","VelenPublicWebCrawler","Yandex"]}
{"domain":"hollywoodreporter.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":26,"stanza_tokens":["*","anthropic-ai","cohere-ai","omgili","omgilibot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ChatGPT","DeepSeek","DeepSeekBot","GPTBot","OAI-SearchBot","ClaudeBot","Claude-Web","Diffbot","FacebookBot","Google-Extended","Meta-ExternalAgent","Meta-ExternalFetcher","PerplexityBot","Timpibot","Webzio-Extended","YouBot","cXensebot"]}
{"domain":"trip.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","AdsBot-Google-Mobile","AdsBot-Google","Googlebot-Image","Twitterbot"]}
{"domain":"adtelligent.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"crunchyroll.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"coinbase.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["facebookexternalhit","Twitterbot","LinkedInBot","WhatsApp","Slackbot","Discordbot","AdsBot-Google","AdsBot-Google-Mobile","*"]}
{"domain":"crazyegg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"al-adtech.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"mpg.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"flaticon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Mediapartners-Google","Adsbot-Google"]}
{"domain":"ztomy.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"azure-dns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tdnsv5.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"edu-servers.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"de.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"livescience.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","AmazonAdBot","bytespider","mistralai","cohere","ai2bot","youbot","omgili","diffbot","kangaroo","img2dataset","amazonbot","amazon-qbusiness","meta-externalagent","meta-webindexer"]}
{"domain":"khanacademy.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ureca-lab.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"urldefense.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"entrust.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"arizona.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"qianxun.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ssa.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE",403]}
{"domain":"sharepointonline.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"sparteo.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"s-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"arc-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ifeng.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"verkada.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lastpass.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iheart.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["AmazonAdBot","Mediapartners-Google","facebookexternalhit","*","Googlebot-News"]}
{"domain":"cymru.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["ia_archiver"]}
{"domain":"msecnd.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"meituan.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"magazineluiza.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"getyourguide.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"joinserver.xyz","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"tagesschau.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":30,"stanza_tokens":["*","webzip","AhrefsBot","webwhacker","websauger","webcapture","teleport","sitesnagger","net attache","httrack","applebot-extended","bytespider","ccbot","claudebot","google-extended","cohere-training-data-crawler","deepseekbot","deepseek","deepseek-crawler","diffbot","facebookbot","meta-externalagent","gptbot","grokbot","img2dataset","laiondownloader","mistralai-crawler","omgili","omgilibot","pangubot"]}
{"domain":"ethz.ch","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"bumlam.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"test-cdn-kkk.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ucl.ac.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Twiceler","oBot","*","Funnelback"]}
{"domain":"zscaler.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dhs.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Petalbot"]}
{"domain":"lgtvsdp.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"slackb.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"thehill.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":39,"stanza_tokens":["*","GPTBot","OAI-SearchBot","PerplexityBot","ia_archiver","Google-Extended","ClaudeBot","Claude-Web","anthropic-ai","CCBot","FacebookBot","proximic","Moreover","AhrefsBot","Applebot-Extended","GumGum","Verity","AwarioRssBot","AwarioSmartBot","Bytespider","ChatGPT-User","cohere-ai","DataForSeoBot","Diffbot","FriendlyCrawler","Google-CloudVertexBot","ImagesiftBot","magpie-crawler","Meta-ExternalAgent","meta-externalagent","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","Quora-Bot","Scrapy","TurnitinBot"]}
{"domain":"mirtesen.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","Googlebot","*"]}
{"domain":"ibb.co","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ccc.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"tmall.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"drupal.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mi-img.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"moodle.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"webrootcloudav.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"gccdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNREFUSED"]}
{"domain":"tutu.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wshareit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"powerbi.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ringcentral.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"slashdot.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gboxlab.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"inmobicdn.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"ionos.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Slurp","msnbot","dotbot","kinshoo"]}
{"domain":"x5.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sureserver.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"ixigua.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Baiduspider","YisouSpider"]}
{"domain":"technologyreview.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":39,"stanza_tokens":["*","Amazonbot","anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","GPTbot","ImagesiftBot","magpie-crawler","Meta-ExternalAgent","meta-externalagent","Meta-ExternalFetcher","meta-externalfetcher","NewsNow","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Quora-Bot","Scrapy","Timpibot","TurnitinBot","YouBot"]}
{"domain":"okx.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Twitterbot","Yeti"]}
{"domain":"protechts.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ycombinator.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","facebookexternalhit","Twitterbot","LinkedInBot"]}
{"domain":"screenconnect.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"theglobeandmail.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":184,"stanza_tokens":["AmazonAdBot","Googlebot-News","AdsBot-Google","*","AddSearchBot","AhrefsBot","AI2Bot","AI2Bot-DeepResearchEval","AI2Bot-Dolma","aiHitBot","AliyunSecBot","Amazonbot","AmazonBuyForMe","amazon-kendra","Amzn-SearchBot","Amzn-User","Andibot","Anthropic-ai","Anomura","archive.org_bot","Archive-It","atlassian-bot","AudigentAdBot","Awario","AwarioRssBot","AwarioSmartBot","BaiduSpider","bedrockbot","bigsur.ai","BLEXBot","Brandwatch","Bravebot","Brightbot 1.0","Bytespider","carbon-umbrella-bot/1.0","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT Agent","ChatGPT-User","Cision","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","Cloudflare-AutoRAG","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Crawlspace","DataForSeoBot","Datenbank Crawler","DeepSeekBot","Deepseek","Devin","Diffbot","DuckAssistBot","EchoboxBot","ErisBot","Factset_spyderbot","fiperbot/0.1","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Gnowit","GnowitNewsbot","Google-CloudVertexBot","Google-Extended","Google-NotebookLM","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","Grok","Heritrix","ia_archiver","iAskBot","iaskspider/2.0","IbouBot","ICC-Crawler","imageSpider","ImagesiftBot","img2dataset","Imprint (netEstate Imprint Crawler)","Info2424.info","ISSCyberRiskCrawler","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","LAIONDownloader","laion-huggingface-processor","LCC","LinerBot","Linguee Bot","linkfluence","LinkupBot","LinkupBot/1.0"]}
{"domain":"olympics.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"lgsmartad.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"hs-banner.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tiktokglobalshopv.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"sketchfab.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","bingbot"]}
{"domain":"network-auth.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wsglb0.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sbcglobal.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"wpsdns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kasserver.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"openwebmp.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"chinadaily.com.cn","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"prezi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot"],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-SearchBot"],"path_restricted":["Claude-User","anthropic-ai"]}],"stanza_token_count":5,"stanza_tokens":["*","ClaudeBot","GPTBot","Claude-SearchBot","OAI-SearchBot"]}
{"domain":"carnet.hr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cachefly.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"kuaishou.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["Baiduspider","Baiduspider-video","Googlebot","YoudaoBot","Sogou web spider","Sogou inst spider","360Spider","bingbot","PerplexityBot","Perplexity-User","GPTBot","ChatGPT-User","OAI-SearchBot","deepseekbot","*"]}
{"domain":"rr.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"csdn.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"woocommerce.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hostingradio.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"fortinet.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"indiegogo.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"distmed.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"urbanairship.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"captcha-delivery.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"scopely.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"usa.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"perl.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["NPBot"]}
{"domain":"browser-intake-us5-datadoghq.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"hindustantimes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","facebookexternalhit","Facebot","Google-Extended","AmazonAdBot","AdsBot-Google","Mediapartners-Google"]}
{"domain":"howstuffworks.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"harman.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","UNABLE_TO_VERIFY_LEAF_SIGNATURE","TimeoutError"]}
{"domain":"isnssdk.com","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":["ENOTFOUND",503,503]}
{"domain":"coloros.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"roskazna.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cve.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"afraid.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"zeit.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":32,"stanza_tokens":["Googlebot-News","*","anthropic-ai","Ai2Bot-Dolma","Applebot-Extended","Baiduspider","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","CloudVertexBot","cohere-training-data-crawler","Cotoyogi","DeepSeekBot","Diffbot","FacebookBot","Google-CloudVertexBot","Google-Extended","GPTBot","GrapeshotCrawler","img2dataset","Kangaroo Bot","KunatoCrawler","Meta-ExternalAgent","PanguBot","Perplexity-User","PerplexityBot","quillbot.com","Spider","TerraCotta","Timpibot","VelenPublicWebCrawler"]}
{"domain":"bt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"julydns.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"dnsv4.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNREFUSED"]}
{"domain":"klarna.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"strpst.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gamespot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"weplayapp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"coe.int","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"northwestern.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gotowebinar.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"tenable.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"smithsonianmag.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"newscientist.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":103,"stanza_tokens":["AhrefsBot","AI2Bot","Ai2Bot-Dolma","Amazonbot","amazon-kendra-","anthropic-ai","Applebot-Extended","bedrockbot","Bytespider","CloudflareBrowserRenderingCrawler","CCBot","ChatGLM-Spider","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","Cotoyogi","DeepSeekBot","Diffbot","DuckAssistBot","EchoboxBot","FacebookBot","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","Grok","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","KunatoCrawler","LinerBot","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","MistralAI-User","OAI-Operator","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","Perplexity-User","PetalBot","QualifiedBot","Scrapy","Seekr","Sidetrade indexer bot","TaraGroup Intelligent Bot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","wpbot","WRTNBot","YouBot","*","archive-it.org","archive.org_bot","ArchiveBot","Arquivo-web-crawler","Authory","babel","bl.uk_ldfc_bot","bl.uk_ldfc_renderbot","bne.es_bot","bnf.fr_bot","ecoresearch","ecoresearchCrawler","Europarchive.org","kb.dk_bot","Kulturarw3","LivelapBot","mediacloud","mirrorweb","MSIECrawler","nationalarchives.gov.uk","netarkivet.dk","netEstate NE Crawler","Nicecrawler","nl-israel_iaharvester2026","nlnbot","nlncrawler","PageFreezer","SafeDNS","SmarshBot","special_archiver","tkl.iis.u-tokyo.ac.jp","TurnitinBot","webarchiv.cz"]}
{"domain":"archives.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","usasearch"]}
{"domain":"loopia.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ttwstatic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cutt.ly","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hubspot.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sensic.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ford.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"raiffeisen.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","Googlebot"]}
{"domain":"padlet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cdnhwc3.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","CERT_HAS_EXPIRED"]}
{"domain":"nheos.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tizen.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"aditude.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot","*"]}
{"domain":"webs.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"medlineplus.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tuyaeu.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"matomo.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unimelb.edu.au","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bigcommerce.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"listdl.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mona.co","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"itv.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nic.media","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"msftstatic.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"rudderstack.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"surfshark.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aibotplatform.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"p-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"brevo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","ia_archiver","archive.org_bot","special_archiver","Heritrix"]}
{"domain":"safedk.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"leparisien.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":210,"stanza_tokens":["GPTBot","Amazonbot","anthropic-ai","CCBot","Claude-web","ClaudeBot","cohere-ai","PerplexityBot","*","Googlebot-Image","Mediapartners-Google","flipboard","FlipboardProxy","5emeRue","5erue","adequat","adequat-systems","Alexibot","AlvinetSpider","AmiSoftware","Antenne Hatena","ApocalXExplorerBot","Argus","Ask n read","asknread.com","asterias","Augure","augure","auramundi","BackDoorBot/1.0","BizInformation","Black Hole","Bloodhound","Bloomberg","BlowFish/1.0","BotALot","BuiltBotTough","Bullseye/1.0","BunnySlippers","Cegbfeieh","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","Cision","coexel","ConveraCrawler","CopyRightCheck","Corporama","cosmos","Crescent","cydralspider","Digimind","DISCo Pump 3.1","DittoSpyder","dotbot","Download Ninja","EDD","ellisphere","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","Europresse","Explore","eureka","ExtractorPro","FacebookBot","Factiva","Fasterfox","Fetch","Flamingo_SearchEngine","Foobot","gammaSpider","grub-client","Harvest/1.5","hloader","httplib","HTTrack","HTTrack 3.0","humanlinks","ia_archiver","ia_archiver-web.archive.org","Igentia","indexer","InfoNaviRobot","infoseek","JennyBot","Jetbot","JikeSpider","k2spider","Kantar","kbcrawl","Kenjin Spider","Knowings","larbin","leadbox","LexiBot","libWeb/clsHTTP","libwww"]}
{"domain":"pxf.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cell.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":22,"stanza_tokens":["*","Twitterbot","LinkedInBot","GPTBot","ChatGPT-User","CCBot","Google-Extended","PerplexityBot","AI2Bot","Amazonbot","anthropic-ai","ClaudeBot","Bytespider","cohere-training-data-crawler","cohere-ai","Diffbot","iAskBot/1.0","meta-externalagent","MistralAI-User","omgili","TimpiBot","YouBot"]}
{"domain":"java.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adguard.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"nhk.or.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":20,"stanza_tokens":["GPTBot","ChatGPT-User","ClaudeBot","Google-Extended","Applebot-Extended","anthropic-ai","cohere-ai","CCBot","ICC-Crawler","Bytespider","OAI-SearchBot","PerplexityBot","Meta-ExternalAgent","PetalBot","baiduspider","baiduimagespider","Googlebot-Image","Googlebot-Video","Twitterbot","*"]}
{"domain":"rostelecom.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"rockstargames.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":8,"stanza_tokens":["ia_archiver","Applebot-Extended","CCBot","cohere-ai","cohere-training-data-crawler","ClaudeBot","GPTBot","Meta-ExternalAgent"]}
{"domain":"elastic.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"poshvine.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"minecraft.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"opera-api2.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"ikarem.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"sl-reverse.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"news.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"alibabacorp.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ECONNREFUSED","TimeoutError"]}
{"domain":"mmstat.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cloudapp.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNREFUSED"]}
{"domain":"lgthinq.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"centos.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["my_agent","*"]}
{"domain":"ibytedtos.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"akaquill.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zefrdata.com","fetch_ok":false,"usable":false,"reason":"other","status":530,"error":null,"attempts":["ENOTFOUND",530,530]}
{"domain":"envato.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":5,"stanza_tokens":["*","GPTBot","Google-Extended","GoogleOther","CCBot"]}
{"domain":"gutenberg.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ETIMEDOUT","ETIMEDOUT"]}
{"domain":"queniuck.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"instacart.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"spb.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","TimeoutError"]}
{"domain":"bizjournals.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":24,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","YahooSeeker/CafeKelsa","Yandex","Mail.Ru/1.0","blp_bbot","Baiduspider","MauiBot","cohere-ai","ia_archiver","ia_archiver-web.archive.org","archive.org_bot","heritrix","Mediapartners-Google","Twitterbot","Facebot"]}
{"domain":"parallels.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","SEOkicks","Mauibot","ZumBot"]}
{"domain":"chicagotribune.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":68,"stanza_tokens":["*","AI2Bot","Ai2Bot-Dolma","Amazonbot","Amzn-SearchBot","anthropic-ai","Applebot","Applebot-Extended","Archive.org_bot","Baiduspider","Brightbot 1.0","Bytespider","carbon-umbrella-bot","CCBot","ChatGLM-Spider","ChatGPT-User","ClaudeBot","Claude-Searchbot","Claude-User","Claude-Web","Cohere-training-data-crawler","Cotoyogi","Dalvik","Datenbank Crawler","Diffbot","DocSpotBot","DuckAssistBot","FacebookBot","GnowitNewsbot","Google-Extended","GPTBot","grok-crawler","ia_archiver-web.archive.org","ICC-Crawler","imageSpider","Kangaroo Bot","laion-huggingface-processor","LCC","Mantisbot","Meta-ExternalAgent","MistralAI-User","netEstate Imprint Crawler","OAI-SearchBot","omgili","PanguBot","PerplexityBot","Perplexity-User","PetalBot","SBIntuitionsBot","SirdataBot","Spider","Thinkbot","Timpibot","VelenPublicWebCrawler","Vivaldi","Webzio-extended","YandexBot","YouBot","AmazonAdBot","facebookexternalhit","grapeshot","Mozilla/5.0 (compatible; EchoboxBot/1.0; hash/w4mwnpbXf3MFAbxOkJRw; +http://www.echobox.com)","Twitterbot","Mozilla/5.0 (compatible; NewsRoom.BI/0.1; +http://www.newsroom.bi/bot.html)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36(KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; mrfCompass-Marshall/1.0)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Jukebox/1.0)"]}
{"domain":"0xrpc.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ECONNRESET",404]}
{"domain":"notion.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","BLEXBot","AhrefsBot","Amazonbot","SemrushBot","dotbot"]}
{"domain":"rumble.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["facebookexternalhit","*"]}
{"domain":"nsw.gov.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kbz0pwvxmv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["GPTBot","PerplexityBot","Google-Extended","anthropic-ai","Claude-Web","*"]}
{"domain":"su89-cdn.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["GPTBot","PerplexityBot","Google-Extended","anthropic-ai","Claude-Web","*"]}
{"domain":"supercdn.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"ksapisrv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["Baiduspider","Baiduspider-video","Googlebot","YoudaoBot","Sogou web spider","Sogou inst spider","360Spider","bingbot","*"]}
{"domain":"addthis.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"edge.apple","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"tiktokpangle-cdn-us.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bkcdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nic.tv","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"tdnsv6.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pbstck.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"jtvnw.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"crowdstrike.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"phantom.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AdsBot-Google","*"]}
{"domain":"visa.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"inmotionhosting.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"podbean.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kohls.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"terra.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"userway.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"army.mil","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["archive.org_bot","*"]}
{"domain":"mojohost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ilo.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"int08h.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"history.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dalyfeds.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wikisource.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["MJ12bot","Mediapartners-Google*","IsraBot","Orthogaffe","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","fast","wget","grub-client","k2spider","NPBot","WebReaper","SemrushBot","*"]}
{"domain":"archlinux.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"flurry.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"heise.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":336,"stanza_tokens":["*","Mediapartners-Google*","GPTBot","ChatGPT-User","OAI-SearchBot","PerplexityBot","Perplexity-User","ClaudeBot","Claude","Claude-SearchBot","Claude-User","Claude-Web","MistralAI-User","5emeRue","5erue","ACQUIRE MEDIA","ACTIV Financial (CME Group)","AddSearchBot","adequat","adequat-systems","AI2Bot","AlphaSense","Amazonbot","AmiSoftware","anthropic-ai","Applebot","Applebot-Extended","Aqua_Products","ArgClrInt","Ask n read","asknread.com","asterias","Augure","auramundi","AwarioRssBot","AwarioSmartBot","b2w/0.1","BackDoorBot/1.0","Barchart.com","BattleFin","Bernin IT","bigsur.ai","Blackboard Safeassign","BlowFish/1.0","BLP_bbot","bluematrix","Bookmark search tool","BotALot","Brandwatch","Briefcase.news","Buck","BuiltBotTough","Bullseye/1.0","BunnySlippers","Bytespider","CCBot","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","CikisiBot","Cision","CloudVertexBot","coexel","cohere-ai","cohere-training-data-crawler","Comtex News Network","ConveraCrawler","Copyright Licensing Agency","CopyRightCheck","Corporama","cosmos","Cotoyogi","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","D&B Hoovers","Data Expression","Data Observer","Dataminr","Datenbank Crawler","Dealogic","Devin","Diffbot","Digimind","DirectFN","DittoSpyder","DuckAssistBot","Dun & Bradstreet","Dun & Bradstreet - D&B ESG Intelligence","Dun & Bradstreet Data Marketplace","Eagle Alpha","ecoresearch","ellisphere","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","ExtractorPro","FacebookBot","FactSet"]}
{"domain":"hisavana.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"douyinpic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"aliexpress-media.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"douyinvod.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ueiwsp.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,404]}
{"domain":"1password.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"paramountplus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"stumbleupon.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"atomile.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"ndcpp-os.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"venturebeat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":30,"stanza_tokens":["*","Googlebot","Bingbot","MSNBot","BingPreview","Slurp","Google-Extended","GPTBot","ClaudeBot","Claude-Web","anthropic-ai","PerplexityBot","TermlyBot","AhrefsBot","SemrushBot","MJ12bot","dotbot","CCBot","Bytespider","YandexBot","Amazonbot","amazon-QBusiness","Applebot-Extended","ImagesiftBot","MotoMinerBot","SentiBot","IRLbot","DF Bot 1.0","proximic","AwarioSmartBot"]}
{"domain":"html-load.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tmdb.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":93,"stanza_tokens":["AddSearchBot","AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","Andibot","anthropic-ai","Applebot","Applebot-Extended","Awario","bedrockbot","bigsur.ai","Brightbot 1.0","Bytespider","CCBot","ChatGPT Agent","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Datenbank Crawler","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","FacebookBot","facebookexternalhit","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","LinerBot","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NovaAct","OAI-SearchBot","omgili","omgilibot","OpenAI","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","Thinkbot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","wpbot","YaK","YandexAdditional","YandexAdditionalBot","YouBot"]}
{"domain":"huntress.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cuny.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["googlebot","bingbot","msnbot","Twitterbot","Slurp","LinkedInBot","DuckDuckBot","SiteimproveBot-Crawler","SemanticScholarBot","facebookexternalhit","Go-http-client","*"]}
{"domain":"pypi.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"histats.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thetimes.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":49,"stanza_tokens":["*","Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News","Google-InspectionTool","GoogleOther","APIs-Google","Mediapartners-Google","DuckDuckBot","Slurp","Twitterbot","Bingbot","adidxbot","MicrosoftPreview","Applebot","AdsBot-Google","AmazonAdBot","facebookexternalhit","Yahoo Ad Monitoring","parse.ly scraper","Screaming Frog SEO Spider","SemrushBot","Botify","OnCrawl","Chrome-Lighthouse","Slackbot-LinkExpanding","Slack-ImgProxy","Slackbot","Snap URL Preview Service","proximic","GumGum-Bot","TTD-Content","Pubmatic","ias_crawler","outbrain","Leikibot","weborama-fetcher","AHC/2.1","SirdataBot","peer39_crawler/1.0","EchoboxBot/1.0","CriteoBot/0.1","AdDefend Site Context Crawler/1.0","SmartologyBot","Dianomi","ChatGPT-User","GPTBot","OAI-SearchBot"]}
{"domain":"cern.ch","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ravm.tv","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"strato.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pemsrv.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hotjar.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"accor.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"pikabu.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Yandex","Googlebot","Mail.RU_Bot"]}
{"domain":"brightcove.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"bytegle.tech","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bcelive.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wipo.int","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ctfassets.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"digistorm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"indianexpress.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"suno.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"virginia.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pubnative.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"easybrain.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mfadsrvr.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"apptracer.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"canonical.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","Google-Extended","meta-externalagent","PerplexityBot","cohere-ai","Bytespider"]}
{"domain":"snssdk.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"evergage.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"auth0app.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"philips.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vkuseraudio.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"iotcplatform.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"dropboxapi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"knt9.xyz","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"glanceapis.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"bcvcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"osu.edu","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"tsinghua.edu.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"spaceweb.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"google.us","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"jampp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ncsu.edu","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":["TimeoutError",202]}
{"domain":"thomsonreuters.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"startappservice.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"match.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"icloud.com.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot","facebookexternalhit"]}
{"domain":"dion.ne.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"beyla.site","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"pstatic.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"trustindex.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"galaxy-cdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"utah.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eurodns.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"drweb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"elastic-cloud.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"steamhistory.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"bitnami.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dc-telemetry.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"tvsquared.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"nextdoor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["Googlebot","Slurp","msnbot","ia_archiver","Teoma","SemrushBot","Seobility","Twitterbot","facebookexternalhit","GPTBot","*"]}
{"domain":"optimove.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"gumroad.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yomiuri.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":47,"stanza_tokens":["wget","Megalodon","ia_archiver","libwww","curl","libwww-perl","Scrapy","Nutch","indexpert","Fess","Mechanize","Java","CCBot","OAI-SearchBot","ChatGPT-User","GPTBot","Google-Extended","GoogleOther","AmazonBot","AmazonAdBot","anthropic-ai","Claude-Web","ICC-Crawler","Applebot-Extended","Bytespider","PerplexityBot","Perplexity-ai","Perplexity-User","ClaudeBot","Claude-SearchBot","Claude-User","cohere-ai","cohere-training-data-crawler","Diffbot","FacebookBot","facebookexternalhit","Gensparkbot","magpie-crawler","meta-externalagent","meta-externalfetcher","facebookcatalog","omgili","omgilibot","Timpibot","Webzio-Extended","bingbot","*"]}
{"domain":"githubcopilot.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"sportradar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rottentomatoes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"sprinklr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"coingecko.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fcc.gov","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ctmail.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"uribl.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"trafficjunky.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"anchor.fm","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"adnxs.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","EHOSTUNREACH"]}
{"domain":"privacyshield.gov","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"alfabank.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"treasury.gov","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"azurecontainerapps.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"infomaniak.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"realtor.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"capitalonegslbex.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ax-dc-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"iop.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-03.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"splashtop.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","SemrushBot","SemrushBot-SA"]}
{"domain":"imageshack.us","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":98,"stanza_tokens":["AddSearchBot","AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","Andibot","anthropic-ai","Applebot","Applebot-Extended","Awario","bedrockbot","bigsur.ai","Brightbot 1.0","Bytespider","CCBot","ChatGPT Agent","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Datenbank Crawler","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","FacebookBot","facebookexternalhit","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","Google-Firebase","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","LinerBot","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","meta-webindexer","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NovaAct","OAI-SearchBot","omgili","omgilibot","OpenAI","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","ShapBot","Sidetrade indexer bot","TerraCotta","Thinkbot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","wpbot","YaK","YandexAdditional","YandexAdditionalBot","YouBot","*"]}
{"domain":"blismedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":17,"stanza_tokens":["LinkLint","W3C-checklink","gsa-crawler","008","Netsparker","*","ClaudeBot","GPTBot","CCBot","meta-externalagent","Bytespider","YisouSpider","Baiduspider","YandexBot","Applebot","Amazonbot","Sogou web spider"]}
{"domain":"phys.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","ClaudeBot","GPTBot","FacebookBot","AhrefsBot","CCbot","Bytespider","cohere-ai","Google-Extended","Applebot-Extended","FriendlyCrawler","ChatGLM-Spider","Scrapy","DataForSeoBot","ImagesiftBot"]}
{"domain":"pullcm.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"sueddeutsche.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":33,"stanza_tokens":["*","backlink-check.de","BacklinkCrawler","ExtractorPro","Fasterfox","LinkextractorPro","LinkWalker","MJ12bot","Openbot","rogerbot","searchpreview","SemrushBot","SEODAT","SEOENGBot","SEOkicks-Robot","True_Robot","URL Control","URL_Spider_Pro","xovi","GPTBot","um-IC","Google-Extended","ClaudeBot","Claude-Web","AhrefsBot","ImagesiftBot","GumGum Bot","anthropic-ai","ChatGPT-User","cohere-ai","PerplexityBot","Amazonbot","AmazonAdBot"]}
{"domain":"shopee.sg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot","Googlebot-Mobile","Bingbot","*"]}
{"domain":"zalo.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["Googlebot"]}
{"domain":"1capp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ventraip.net.au","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"undp.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"dssott.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fastly-masque.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hostedemail.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"nokia.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"joinmassive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vogue.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","FacebookExternalHit","Twitterbot","Pinterestbot","LinkedInBot","TikTokSpider","Storebot-Google","Google-InspectionTool","AmazonAdBot","Google-Extended","Google-CloudVertexBot","GoogleOther","Applebot-Extended","Amazonbot","meta-webindexer","meta-externalagent","meta-externalfetcher","ClaudeBot","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","cohere-training-data-crawler","cohere-ai","CCBot","PanguBot","PetalBot","Bytespider","Diffbot","DuckAssistBot","MistralAI-User","Timpibot","Webzio","Webzio-Extended","archive.org_bot","ia_archiver","ia_archiver-web.archive.org","heritrix"]}
{"domain":"ciscospark.com","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":["TimeoutError",503,503]}
{"domain":"ubisoft.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"allawntech.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",200]}
{"domain":"cookieyes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","MJ12bot","PetalBot","SemrushBot","Heritrix","TurnitinBot","Turnitin"]}
{"domain":"watchguard.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"incognia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kochava.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","OAI-SearchBot"]}
{"domain":"pullcf.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"panasonic.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"google.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"faz.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":49,"stanza_tokens":["*","Meltwater","NewsNow","Bloodhound","cydralspider","downloadexpress","gammaSpider","ObjectsSearch","Pimptrain","Raven","wapspider","WebZinger","Fasterfox","sentibot","GPTBot","omgili","omgilibot","CCBot","Google-Extended","Bytespider","anthropic-ai","Diffbot","PerplexityBot","YouBot","Applebot-Extended","ClaudeBot","Claude-Web","Meta-ExternalAgent","DeepSeekBot","DeepSeek","GoogleOther","Claude-SearchBot","Claude-User","cohere-ai","cohere-training-data-crawler","Amazonbot","Timpibot","AI2Bot","DuckAssistBot","Kangaroo Bot","PanguBot","MistralAI-User","Devin","PetalBot","bigsur.ai","CloudVertexBot","LinerBot","OAI-SearchBot","ChatGPT-User"]}
{"domain":"xnxx-cdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"rocketmoney.dev","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"narod.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["ia_archiver","SolomonoBot","SemrushBot","AhrefsBot","AhrefsBot/5.0","MJ12bot","dotbot","dotbot/1.1","rogerbot","linkdexbot/2.0","Genieo","Baiduspider","Baiduspider-video","Baiduspider-image","*"]}
{"domain":"texas.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"webflow.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"swishapps.ai","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"dtscout.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"kakaocdn.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["GoogleOther","*"]}
{"domain":"growone.sg","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"libero.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":42,"stanza_tokens":["*","OAI-SearchBot","BLEXBot","Bloglines/3.1","cityreview","DOC","dotbot","Download Ninja","Exabot","Fetch","grapeshot","grub-client","HTTrack","Jyxobot/1","k2spider","larbin","libwww","linko","Microsoft.URL.Control","MSIECrawler","NPBot","Offline Explorer","PetalBot","proximic","psbot","SemrushBot","sitecheck.internetseer.com","SiteSnagger","Speedy","Teleport","TeleportPro","UbiCrawler","WebCopier","WebReaper","WebStripper","WebZIP","wget","Xenu","Zao","Zealbot","ZyBORG","AmazonAdBot"]}
{"domain":"franceinfo.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":26,"stanza_tokens":["*","Googlebot-Image","Googlebot-News","Flipboard","Amazonbot","Applebot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","FacebookBot","GPTBot","Google-Extended","Meta-ExternalAgent","MistralAI-User","OAI-SearchBot","Perplexity-User","PerplexityBot","anthropic-ai","cohere-ai","cohere-training-data-crawler","AhrefsBot"]}
{"domain":"max.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oaistatic.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,530]}
{"domain":"moz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["rogerbot","AhrefsBot","GPTBot","DDM-DCipher","*"]}
{"domain":"wa.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kpmg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["cludo","*","YouDaoBot"]}
{"domain":"mastodon.social","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":2,"stanza_tokens":["GPTBot","*"]}
{"domain":"ipapi.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aarp.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","Bytespider","DeepSeekBot","CCBot","YisouSpider","Baiduspider","Sogou web spider"]}
{"domain":"tamu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nexx360.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"website-files.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dot.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"shiabank.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","ETIMEDOUT"]}
{"domain":"myq-cloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"instructables.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Mediapartners-Google*","Slurp","AdsBot-Google*","*"]}
{"domain":"yumpu.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"asahi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":37,"stanza_tokens":["*","Googlebot","CCBot","OAI-SearchBot","ChatGPT-User","GPTBot","Google-Extended","Google-CloudVertexBot","ICC-Crawler","anthropic-ai","ClaudeBot","Claude-Web","Claude-SearchBot","Claude-User","Applebot-Extended","cohere-ai","Cohere-training-data-crawler","omgili","omgilibot","PerplexityBot","Perplexity-ai","Perplexity-User","FacebookBot","Meta-ExternalAgent","Meta-externalfetcher","Bytespider","Gensparkbot","AmazonBot","Diffbot","Magpie-crawler","Scrapy","Timpibot","Webzio-Extended","SBIntuitionsBot","SBIntuitions-SearchBot","OpenindexSpider","AI2Bot"]}
{"domain":"bsgslb.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"mookie1.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","ENOTFOUND"]}
{"domain":"x-cdn-static.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["GPTBot","PerplexityBot","Google-Extended","anthropic-ai","Claude-Web","*"]}
{"domain":"today.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":36,"stanza_tokens":["*","Amazonbot","AmazonBot","anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","Claude-User","Claude-SearchBot","cohere-ai","DataForSeoBot","Diffbot","FacebookBot","Google-Extended","GPTBot","magpie-crawler","Meta-ExternalAgent","meta-externalagent","MistralAI-User","NagetBot","NewsNow","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Perplexity-User","Scrapy","TurnitinBot"]}
{"domain":"ko-fi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"capitalone.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":31,"stanza_tokens":["Bingbot","msnbot","DuckDuckBot","Mediapartners-Google","Adsbot-Google","Googlebot","Googlebot-Mobile","Googlebot-Image","ia_archiver","SpeedySpider","TwitterBot","YandexAntivirus/2.0","YandexBot/3.0","YandexImageResizer/2.0","YandexImages/3.0","YandexMedia/3.0","YandexPagechecker/1.0","YandexWebmaster/2.0","YandexZakladki/3.0","Abonti","Aboundexbot","Acoon","Allloadin Favicon Bot","Blekkobot","Daum","Gobuster","semaltbot","Tarmot Gezgin","tchelebi","UkrNet Mail Proxy","*"]}
{"domain":"airtable.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ssl.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mobolize.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ihc.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"irishtimes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":4,"stanza_tokens":["ChatGPT-User","GPTBot","Googlebot-News","*"]}
{"domain":"toutiao.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"amplifyapp.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dnsv.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fbi.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Googlebot","Bingbot","Slurp","DuckDuckBot"]}
{"domain":"tds.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":44,"stanza_tokens":["*","ADmantX","AlphaBot","anthropic-ai","AwarioRssBot","AwarioSmartBot","BLEXBot","Buzzbot","Bytespider","CCBot","ChatGPT-User","claritybot","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","huggingface","ImagesiftBot","img2dataset","magpie-crawler","Meltwater","Neevabot","news-please","NewsNow","Nutch","omgili","omgilibot","panscient.com","Perplexity-ai","PerplexityBot","PetalBot","PiplBot","scoop.it","Scrapy","Seekr","SentiBot","SeznamBot","TurnitinBot","YouBot","ZumBot"]}
{"domain":"foursquare.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":22,"stanza_tokens":["*","Branch Metrics API","Branch API","PiplBot","MJ12bot","CCBot","SeznamBot","Exabot","netseer","Mappy","crawler4j","Gigabot","ZoomBot","BUbiNG","Getintent Crawler","BLEXBot","trendictionbot","hyscore","magpie-crawler","RyteBot","GPTBot","ChatGPT-User"]}
{"domain":"datadoghq-browser-agent.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"heart.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rutgers.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uga.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ps.kz","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rosminzdrav.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"foggymorningmist.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mysecurecloudhost.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"sfmc-marketing.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"quillbot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unisoc.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ECONNREFUSED",200]}
{"domain":"libsyn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cox.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"thehindu.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"france24.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":326,"stanza_tokens":["*","Googlebot","AI2Bot","Ai2Bot-Dolma","AlibabaBot","AliyunBot","anthropic-ai","Baidu-YunGuanCe","Big Sur AI","ByteDance-AI","ChatGPT Operator","ChatGPT-User","Claude-Code","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","DeepSeekBot","DigitalOceanGenAICrawler","DuckAssistBot","ERNIEBot","ERNIE-Bot","Google-Extended","GPTBot","Grok","img2dataset","Kimibot","KimiCrawler","LINER Bot","Meta Llama","MetaAIBot","Moonshot-AI","MoonshotBot","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","OAI-SearchBot","PanguBot","Perplexity-User","Perplexity-User/1.0","PerplexityBot","quillbot.com","QwenBot","SBIntuitionsBot","TongyiBot","Tongyi-Crawler","YiyanBot","YouBot","008","5emeRue","5erue","adequat","adequat-systems","ADmantX bot","AhrefsBot","Alexibot","AlvinetSpider","AmiSoftware","Antenne Hatena","ApocalXExplorerBot","Argus","Ask n read","asknread.com","asterias","Augure","augure","auramundi","Babya Discoverer","BackDoorBot/1.0","BacklinkCrawler","bitlybot","BizInformation","Black Hole","Blekkobot","BLEXBot","Bloodhound","BlowFish/1.0","BotALot","BuiltBotTough","Bullseye/1.0","BunnySlippers","CCBot","Cegbfeieh","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","Cision","Cliqzbot","coexel","ConveraCrawler","CopyRightCheck","Corporama","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","cydralspider","Digimind"]}
{"domain":"grok.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":7,"stanza_tokens":["*","GPTBot","ChatGPT-User","PerplexityBot","ClaudeBot","Google-Extended","Applebot-Extended"]}
{"domain":"senderscore.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"discovery.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Mediapartners-Google","Pinterest"]}
{"domain":"nic.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AdsBot-Google-Mobile","AdsBot-Google","PetalBot","dotbot","AhrefsBot"]}
{"domain":"thawte.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mydramawave.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"saby.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","Googlebot","Googlebot-Image","YandexBot","YandexImages","Bingbot","Applebot","DuckDuckBot","Baiduspider","facebot","Slurp"]}
{"domain":"netis.cc","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cc-ecdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pphosted.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_SSL_TLSV1_UNRECOGNIZED_NAME"]}
{"domain":"coupangcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"baifentan.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"s-microsoft.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"medscape.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"riotcdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tableau.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"1drv.ms","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uspto.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zapier.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","facebookexternalhit","Baiduspider","GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","Google-Extended","PerplexityBot","DuckAssistBot"]}
{"domain":"crealitycloud.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","GPTBot","Google-Extended","anthropic-ai","CCBot","PerplexityBot","YouBot"]}
{"domain":"w55c.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"boyuanhulian.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rsc.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["gsa-crawler","Guidebot","*","Slurp","AhrefsBot","AhrefsSiteAudit"]}
{"domain":"cn.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"redbubble.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"dns.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","EHOSTUNREACH"]}
{"domain":"tidal.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pastebin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"queniucf.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"iea.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"flightradar24.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["ChatGPT-User"],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":5,"stanza_tokens":["*","CCBot","GPTBot","ChatGPT-User","AmazonAdBot"]}
{"domain":"hybrid.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"drmgms.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"citrix.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dnspod.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"square.site","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jpush.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ti.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"redbull.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Slurp"]}
{"domain":"globalcdn.co","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"coca-cola.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tcylgslb.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"a.co","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"mystrikingly.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"proton.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"system-monitor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pandasecurity.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dnse4.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"indazn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ucsb.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"assetsconfigcdn.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"ifconfig.me","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"powerapps.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"c6gj-static.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["GPTBot","PerplexityBot","Google-Extended","anthropic-ai","Claude-Web","*"]}
{"domain":"imgix.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sinaimg.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ohchr.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"tealiumiq.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"theregister.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":68,"stanza_tokens":["Googlebot","Bingbot","Slurp","DuckDuckBot","Baiduspider","YandexBot","Applebot","OAI-SearchBot","ChatGPT-User","PerplexityBot","Claude-User","Claude-SearchBot","claude-code","AdsBot-Google","AdsBot-Google-Mobile","Mediapartners-Google","facebookexternalhit","Twitterbot","LinkedInBot","Pinterestbot","Slackbot","Slackbot-LinkExpanding","Discordbot","TelegramBot","WhatsApp","ias_crawler","ias_wombles","ias_admantx","DoubleVerifyBot","DVBot","proximic","peer39_crawler","GrapeshotCrawler","grapeshot","GumGum","GlancelotBot","TTD-Content","PubMatic Crawler Bot","SinceraSyntheticUser","AmazonAdBot","Leikibot","SlickBot","GPTBot","ClaudeBot","anthropic-ai","Claude-Web","Google-Extended","Google-CloudVertexBot","Meta-ExternalAgent","Meta-ExternalFetcher","FacebookBot","Applebot-Extended","Amazonbot","Bytespider","CCBot","cohere-ai","Diffbot","YouBot","DuckAssistBot","Timpibot","Omgilibot","Omgili","ImagesiftBot","GrokBot","AI2Bot","FirecrawlAgent","Scrapy","*"]}
{"domain":"usertrust.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","TimeoutError"]}
{"domain":"synology.me","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"networklinks.space","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"redis.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tenda.com.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"websupport.sk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"newsbreak.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":31,"stanza_tokens":["*","anthropic-ai","AmazonBot","Claude-Web","cohere-ai","ClaudeBot","ChatGPT-User","GPTBot","CCBot","Google-Extended","DataForSeoBot","Diffbot","FacebookBot","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Scrapy","TurnitinBot","Applebot-Extended","Meta-ExternalAgent","meta-externalagent","OAI-SearchBot","Bytespider","Bard","Claude","Anthropic"]}
{"domain":"faqs.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_PACKET_LENGTH_TOO_LONG","attempts":["ERR_SSL_PACKET_LENGTH_TOO_LONG","ERR_SSL_PACKET_LENGTH_TOO_LONG"]}
{"domain":"chip.de","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"starmakerstudios.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Bytespider","DataForSeoBot","Amazonbot","AliyunSecBot","*"]}
{"domain":"haxx.se","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"wanadoo.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jwpltx.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cdnhwczba04.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"akadns99.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hakunaymatata.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"merlincdn.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"byteimg.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mapquest.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":43,"stanza_tokens":["AmazonAdBot","*","AI2Bot","AI2Bot-Dolma","Amazonbot","AndiBot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ChatGPT-User/2.0","claudebot","ClaudeBot","Claude-Web","cohere-ai","Cohere-ai","Diffbot","DiffBot","ExaBot","FacebookBot","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","GPTBot","Img2dataset","ImagesiftBot","magpie-crawler","Meta-ExternalAgent","meta-externalagent","Meta-ExternalFetcher","OAI-SearchBot","omgili","Omgilibot","PanguBot","PerplexityBot","PerplexityBot-User","PetalBot","PhindBot","SentiBot","Timpibot","webzio-extended","YouBot"]}
{"domain":"namecheap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cancer.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ctvnews.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":12,"stanza_tokens":["*","Googlebot-Image","Twitterbot","GPTBot","ChatGPT-User","OAI-SearchBot","PerplexityBot","CCBot","Anthropic-ai","DeepSeek","DeepSeekBot","Claude-Web"]}
{"domain":"ru.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"web.app","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"pavietnam.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"laracasts.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"argotunnel.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"unmsapp.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"traveloka.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":22,"stanza_tokens":["*","Google-Extended","Gemini-Deep-Research","GPTBot","OAI-SearchBot","ChatGPT-User","PerplexityBot","Perplexity-User","ClaudeBot","Claude-User","Claude-SearchBot","Applebot-Extended","meta-externalagent","meta-webindexer","Bytespider","Amazonbot","YandexBot","AdsBot-Google","Google-HotelAdsVerifier","Google-HotelAdsVerifier/2.0","adsnaver","Yeti"]}
{"domain":"sbis.ru","fetch_ok":false,"usable":false,"reason":"other","status":410,"error":null,"attempts":[410,410]}
{"domain":"webflow.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TypeError",200]}
{"domain":"univie.ac.at","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hscollectedforms.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"metro.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":103,"stanza_tokens":["*","archive-it.org","archive.org_bot","ArchiveBot","Arquivo-web-crawler","Authory","babel","bl.uk_ldfc_bot","bl.uk_ldfc_renderbot","bne.es_bot","bnf.fr_bot","ecoresearch","ecoresearchCrawler","Europarchive.org","kb.dk_bot","Kulturarw3","LivelapBot","mediacloud","mirrorweb","MSIECrawler","nationalarchives.gov.uk","netarkivet.dk","netEstate NE Crawler","Nicecrawler","nl-israel_iaharvester2026","nlnbot","nlncrawler","PageFreezer","SafeDNS","SmarshBot","special_archiver","tkl.iis.u-tokyo.ac.jp","TurnitinBot","webarchiv.cz","WordPress.com mShots","XY-Archive-Compliance-Archiver","XY-Archive-Compliance-Crawler","AhrefsBot","AI2Bot","Ai2Bot-Dolma","Amazonbot","amazon-kendra-","anthropic-ai","Applebot-Extended","bedrockbot","Bytespider","CloudflareBrowserRenderingCrawler","CCBot","ChatGLM-Spider","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","Cotoyogi","DeepSeekBot","Diffbot","DuckAssistBot","EchoboxBot","FacebookBot","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","Grok","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","KunatoCrawler","LinerBot","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","MistralAI-User","OAI-Operator","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","Perplexity-User","PetalBot","QualifiedBot","Scrapy","Seekr","Sidetrade indexer bot","TaraGroup Intelligent Bot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended"]}
{"domain":"cpan.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","The Knowledge AI"]}
{"domain":"v-videoapp.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"4wps.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","attempts":["ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","ENOTFOUND"]}
{"domain":"wixdns.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"salesforce-scrt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["googlebot","*"]}
{"domain":"intelbras.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":11,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","AhrefsBot"]}
{"domain":"fortinet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["CCBot/2.0","Google-CloudVertexBot","GoogleOther","AhrefsBot","SemrushBot","Baiduspider","BingBot","Brightedge","ChatGPT-User","Claude-SearchBot","Claude-User","Exabot","FeedFetcher-Google","Google-extended","Google-InspectionTool","Google-Pinpoint","Google-Site-Verification","Google-speakr","GoogleBot","Googlebot-Image","Googlebot-News","Googlebot-Video","GPTBot","LinkedInBot","OAI-SearchBot","PerplexityBot","SchemaBot/1.2","Twitterbot","facebookexternalhit","Qwantify","MojeekBot","PetalBot","YouBot","*"]}
{"domain":"vrbo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["*","google-hoteladsverifier","Mediapartners-Google","AdsBot-Google-Mobile","AdsBot-Google","AdIdxBot","SemrushBot","OAI-SearchBot","ChatGPT-User","PerplexityBot","Perplexity-User","Claude-User","Claude-SearchBot"]}
{"domain":"zynga.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"magnit.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"caniuse.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"chase.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"calameo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["Linguee","MJ12bot","facebookexternalhit","Twitterbot","LinkedInBot","Amazonbot","Mediapartners-Google","*"]}
{"domain":"zeasn.tv","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"awsdns-29.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"identrust.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"esquire.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AmazonAdBot","Google-InspectionTool","ChatGPT-User","GPTBot","OAI-SearchBot","*"]}
{"domain":"thedailybeast.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"miitbeian.gov.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"EAI_AGAIN","attempts":["EAI_AGAIN","EAI_AGAIN"]}
{"domain":"ntppool.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","AhrefsBot","PetalBot","Claude-User"]}
{"domain":"blxrbdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"navy.mil","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"wellsfargo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"stackdns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dmoz.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"centralnic.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"brookings.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"collegeboard.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tubitv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["grapeshot","*"]}
{"domain":"intercomcdn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"imagemagick.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ubereats.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","googlebot"]}
{"domain":"tokopedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"judns.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"mcgill.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Lucidworks-Anda/2.0","Elastic-Crawler","*","archive.org_bot"]}
{"domain":"cnil.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hellotalk8.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hdslb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dictionary.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":7,"stanza_tokens":["GPTBot","CCBot","deepcrawl","seerbot/1.0","Applebot-Extended","*","AmazonAdBot"]}
{"domain":"unrealengine.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"flic.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"foodnetwork.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":9,"stanza_tokens":["*","Mediapartners-Google","Pinterest","GPTBot","ChatGPT-User","CCBot","ClaudeBot","PerplexityBot","Googlebot-News"]}
{"domain":"google.at","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"mihanwebhost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"index.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":12,"stanza_tokens":["*","Kantar","GPTBot","ChatGPT-User","Google-Extended","CCBot","anthropic-ai","PerplexityBot","Applebot-Extended","Omgilibot","Bytespider","YouBot"]}
{"domain":"yg5sjx5kzy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["GPTBot","PerplexityBot","Google-Extended","anthropic-ai","Claude-Web","*"]}
{"domain":"joomla.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fool.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"flashbots.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"zeotap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dbankedge.cn","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"honeygain.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"line-apps.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"mojang.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"gihc.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"de.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"nic.de","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"kunlunsl.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"tesla.services","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"globalpaymentsinc.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_VERIFY_LEAF_SIGNATURE","attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE","TimeoutError"]}
{"domain":"tmz.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"xhamsterlive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"faipod.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"elle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AmazonAdBot","Google-InspectionTool","ChatGPT-User","GPTBot","OAI-SearchBot","*"]}
{"domain":"aviasales.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"curl.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"llamarpc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"afrinic.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","ETIMEDOUT"]}
{"domain":"lenta.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","GoogleBot","*"]}
{"domain":"awsdns-11.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"allstate.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nasdaq.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"microsoftonline.us","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"_wildcard_.ph","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","TimeoutError"]}
{"domain":"dnse5.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"mxptint.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"unifi-ai.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bybit.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"openldap.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"atsenergo.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"angelfire.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ed.ac.uk","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bet365.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","UND_ERR_CONNECT_TIMEOUT","TimeoutError"]}
{"domain":"affirm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cancer.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mncdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cnrs.fr","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"microsoftazuread-sso.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"sonos.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"walgreens.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"nexon.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"demandbase.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot","*"]}
{"domain":"bttrack.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"newsroom.bi","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"merkle.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"check24.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","TurnitinBot","MauiBot"]}
{"domain":"rkdms.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"loom.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":3,"stanza_tokens":["ia_archiver","GPTBot","*"]}
{"domain":"duosecurity.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"crunchbase.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":16,"stanza_tokens":["*","CCBot","ChatGPT-User","OAI-SearchBot","GPTBot","Google-Extended","anthropic-ai","Omgilibot","Omgili","FacebookBot","Diffbot","Bytespider","ImagesiftBot","cohere-ai","Claude-Web","PerplexityBot"]}
{"domain":"vimeocdn.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"shopee.ph","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot","Googlebot-Mobile","Bingbot","*"]}
{"domain":"grindr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"grass.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ucoz.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"likeevideo.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"feishu.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"pitt.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"flipboard.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":[],"path_restricted":["ChatGPT-User"]}],"stanza_token_count":197,"stanza_tokens":[".ai","Agentic","AI Article Writer","AI Content Detector","AI Dungeon","AI Search Engine","AI SEO Crawler","AI Writer","AI21 Labs","AI2Bot","AIBot","AIMatrix","AISearchBot","AI Training","AITraining","Alexa","Alpha AI","AlphaAI","Amazon Bedrock","Amazon-Kendra","Amazon Lex","Amazon Comprehend","Amazon Sagemaker","Amazon Silk","Amazon Textract","AmazonBot","Amelia","AndersPinkBot","Anthropic","AnyPicker","Anyword","Aria Browse","Articoolo","Automated Writer","AwarioRssBot","AwarioSmartBot","Azure","BardBot","Brave Leo","ByteDance","Bytespider","CatBoost","CC-Crawler","CCBot","ChatGLM","Chinchilla","Claude","ClearScope","Cohere","Common Crawl","CommonCrawl","Content Harmony","Content King","Content Optimizer","Content Samurai","ContentAtScale","ContentBot","Contentedge","Conversion AI","Copilot","CopyAI","Copymatic","Copyscape","Cotoyogi","CrawlQ AI","Crawlspace","Crew AI","CrewAI","DALL-E","DataForSeoBot","DataProvider","DeepAI","DeepL","DeepMind","DeepSeek","Diffbot","Doubao AI","DuckAssistBot","FacebookBot","FacebookExternalHit","Falcon","Firecrawl","Flyriver","Frase AI","FriendlyCrawler","Gemini","Gemma","GenAI","Genspark","GLM","Goose","GPT","Grammarly","Grendizer","Grok","GT Bot","GTBot","Hemingway Editor","Hugging Face","Hypotenuse AI"]}
{"domain":"vanguard.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yahoo.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":46,"stanza_tokens":["*","ADmantX","AlphaBot","anthropic-ai","AwarioRssBot","AwarioSmartBot","BLEXBot","Buzzbot","Bytespider","CCBot","ChatGPT-User","claritybot","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","huggingface","ImagesiftBot","img2dataset","magpie-crawler","Meltwater","Neevabot","news-please","NewsNow","Nutch","omgili","omgilibot","panscient.com","Perplexity-ai","PerplexityBot","PetalBot","PiplBot","scoop.it","Scrapy","Seekr","SentiBot","SeznamBot","TurnitinBot","YouBot","ZumBot","Claude-SearchBot","OAI-SearchBot"]}
{"domain":"twc.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"dmxleo.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"microsoft.us","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dol.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uk.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"ultradns.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ECONNREFUSED"]}
{"domain":"sfmc-content.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"miniclippt.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ns.ua","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"a2hosting.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"databricks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"billboard.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":26,"stanza_tokens":["*","anthropic-ai","cohere-ai","omgili","omgilibot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ChatGPT","DeepSeek","DeepSeekBot","GPTBot","OAI-SearchBot","ClaudeBot","Claude-Web","Diffbot","FacebookBot","Google-Extended","Meta-ExternalAgent","Meta-ExternalFetcher","PerplexityBot","Timpibot","Webzio-Extended","YouBot","cXensebot"]}
{"domain":"ifood.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.com.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"google.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"modrinth.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"mountain.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"si.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","PerplexityBot","Google-Extended"]}
{"domain":"samsungconsent.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cosmopolitan.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AmazonAdBot","Google-InspectionTool","ChatGPT-User","GPTBot","OAI-SearchBot","*"]}
{"domain":"konami.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zemanta.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thrtle.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"shopee.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"bidease.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"ispot.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"allrecipes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"glance.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hootsuite.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"youronlinechoices.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hotmart.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"admatic.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fotmob.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot","Qwantbot","Bingbot"]}
{"domain":"directnic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"n-able.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":62,"stanza_tokens":["*","aiHitBot","Barkrowler","BDCbot","BLEXBot","BLP_bbot","brokenlinkcheck.com","Buck","CCBot","Cliqzbot","cyencebot","DomainCrawler","Dow Jones Searchbot","Exabot","ExtLinksBot","FemtosearchBot","Fever","GarlikCrawler","Gigabot","gobuster","GrapeshotCrawler","heritrix","istellabot","Jersey","Jobkicks","libwww-perl","linkdexbot","LinkpadBot","ltx71 - (http://ltx71.com/)","lua-resty-http","LumtelBot","magpie-crawler","Magus bot","Mail.RU_Bot","Megaindex.ru","Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com","Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com","NL-Crawler","OnPageBot","Riddler","ScoutJet","Scrapy","Seekport","SeznamBot","Siteimprove","SMTBot","UptimeRobot","VelenPublicWebCrawler","Wget","yacybot","Yeti","YisouSpider","YunSecurityBot","ZoominfoBot","AhrefsBot","AhrefsSiteAudit","Caliperbot","DotBot","HubSpot","MJ12bot","rogerbot","SemrushBot"]}
{"domain":"starbucks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["MJ12bot","*"]}
{"domain":"geeksforgeeks.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":6,"stanza_tokens":["*","CCBot","anthropic-ai","Applebot","cohere-ai","Bytespider"]}
{"domain":"brandbucket.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.com.tw","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"amxrtb.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"onaudience.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mediawiki.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["MJ12bot","Mediapartners-Google*","IsraBot","Orthogaffe","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","fast","wget","grub-client","k2spider","NPBot","WebReaper","SemrushBot","*"]}
{"domain":"eastmoney.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"uhc.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"novell.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","EasouSpider","NaverBot","NaverBot-1.0","AhrefsBot"]}
{"domain":"ex.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sfu.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"acuityplatform.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"walkme.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ohmy.bid","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"weibocdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"servenobid.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"lifehacker.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":109,"stanza_tokens":["AddSearchBot","AI2Bot","Ai2Bot-DeepResearchEval","Ai2Bot-Dolma","AmazonBuyForMe","Amazonbot","Amzn-SearchBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot","Applebot-extended","atlassian-bot","AutoRAG","AwarioSmartBot","AzureAI-SearchBot","Big Sur AI","bigsur.ai","Brandwatch","Bravebot","Brightbot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","Datenbank-Crawler","DeepSeekBot","Devin","Diffbot","Direqt Anomura","DuckAssistBot","Echobot Bot","ExaBot","FacebookBot","Factset_spyderbot","FirecrawlAgent","GPTBot","iAsk","iAskBot","iaskspider","ICC Crawler","ICC-Crawler","ImageSiftBot","imageSpider","Kangaroo Bot","Kangaroo-Bot","KlaviyoAIBot","Kunato","laion-huggingface-processor","LCC","LinerBot","LinkupBot","LINER Bot","Manus-User","meta-externalagent","meta-externalfetcher","meta-webindexer","mistral.ai","MistralAI-User","netEstate Imprint Crawler","Novellum AI Crawl","Novellum","NovaAct","OAI-SearchBot","omgili","opencode","PanguBot","Perplexity-User","PerplexityBot","PeopleInc-DCipher-Scraper/1.1.0","PetalBot","Poggio-Citations","QualifiedBot","SeekrBot","SemrushBot-OCOB","SemrushBotSwa","ShapBot","SBIntuitionsBot","Spider","TerraCotta","Timpibot","TavilyBot","Trae","TwinAgent","Velen Crawler","VelenPublicWebCrawler","Webzio-Extended","Wrtn","YouBot"]}
{"domain":"acoustic-adm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"webcontentassessor.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"crm.dev","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"nyc.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nip.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"rackcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ning.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"justia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awmdm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"freshdesk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"paypal.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Twitterbot","facebookexternalhit","*"]}
{"domain":"fastclick.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNREFUSED"]}
{"domain":"usembassy.gov","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TypeError","attempts":["TypeError","TypeError"]}
{"domain":"nymag.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":["GPTBot"],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":108,"stanza_tokens":["Mediapartners-Google","TruliaBot","Twitterbot","*","Mediapartners-Google*","Amazonbot","Applebot","GPTBot","A6-Indexer","AhrefsBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot-Extended","Baiduspider","betaBot","BLEXBot","botify","brands-bot-logo","Brightbot","Bytespider","calculon","CCBot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","collection@infegy.com","Cotoyogi","Crawl4AI","CrunchBot","deepcrawl","DeepSeekBot","Diffbot","DirBuster","DTS","DuckAssistBot","EasouSpider","FacebookBot","FirecrawlAgent","FriendlyCrawler","GlutenCrawler","Google-CloudVertexBot","Google-Extended","GrapeshotCrawler","iaskspider","ICC-Crawler","ImagesiftBot","img2dataset","JackCrawl","Jetslide","linkapedia_bot","Mediatoolkitbot","MegaIndex","meta-externalagent","meta-externalfetcher","Microsoft BITS","Microsoft URL Control","Microsoft-WebDAV-MiniRedir","MistralAI-User","news-please","Novellum","NTENTbot","Nutch","omgili","Omgilibot","OpenLinkProfiler","Perplexity-User","PerplexityBot","PetalBot","Primalbot","python-requests","Re-Animator","Riddler","SafeDNSBot","SBIntuitionsBot","Scrapy","SearchAtlas","SemanticScholarBot","SemrushBot","SentiBot","Sogou","Spider-Pig","sqlmap","StorygizeBot","Test Certificate Info","TikTokSpider","Timpibot","Turnitin","uipbot"]}
{"domain":"tomshardware.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","AmazonAdBot","bytespider","mistralai","cohere","ai2bot","youbot","omgili","diffbot","kangaroo","img2dataset","amazonbot","amazon-qbusiness","meta-externalagent","meta-webindexer"]}
{"domain":"charter.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"goodhousekeeping.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AmazonAdBot","Google-InspectionTool","ChatGPT-User","GPTBot","OAI-SearchBot","*"]}
{"domain":"linklist.bio","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"esa.int","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","YahooCacheSystem","Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)","YandexBot","YandexMedia","YandexImages","YandexCatalog","YaDirectFetcher","YandexBlogs","YandexNews","YandexPagechecker","YandexMetrika","YandexMarket","YandexCalendar","sentibot","bingbot"]}
{"domain":"rqtrk.eu","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"telefonica.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"guard.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iz.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ahacdn.me","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID",404]}
{"domain":"hoster.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","*"]}
{"domain":"fang.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-37.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"adition.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"profi.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"liveintent.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"caltech.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["SemrushBot","BLP_bbot","*"]}
{"domain":"keenetic.pro","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"rian.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"egnyte.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["*","SiteAuditBot","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SemrushBot-CT","SplitSignalBot","SemrushBot-COUB","AhrefsBot","PanzuraVsEgnyte-CompetitiveGuide","NasuniVsEgnyte-CompetitiveGuide","HIPAA-Statement","Cybersecurity-Trends-for-Mid-Sized-Organizations"]}
{"domain":"canva.site","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"xe.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":82,"stanza_tokens":["*","Rogerbot","Exabot","MJ12bot","Dotbot","Gigabot","AhrefsBot","BlackWidow","ChinaClaw","Custo","DISCo","Download Demon","eCatch","EirGrabber","EmailSiphon","EmailWolf","Express WebPictures","ExtractorPro","EyeNetIE","FlashGet","GetRight","GetWeb!","Go!Zilla","Go-Ahead-Got-It","GrabNet","Grafula","HMView","HTTrack","Image Stripper","Image Sucker","Indy Library","InterGET","Internet Ninja","JetCar","JOC Web Spider","larbin","LeechFTP","Mass Downloader","MIDown tool","Mister PiX","Navroad","NearSite","NetAnts","NetSpider","Net Vampire","NetZIP","Octopus","Offline Explorer","Offline Navigator","PageGrabber","Papa Foto","pavuk","pcBrowser","RealDownload","ReGet","SiteSnagger","SmartDownload","SuperBot","SuperHTTP","Surfbot","tAkeOut","Teleport Pro","VoidEYE","Web Image Collector","Web Sucker","WebAuto","WebCopier","WebFetch","WebGo IS","WebLeacher","WebReaper","WebSauger","Website eXtractor","Website Quester","WebStripper","WebWhacker","WebZIP","Wget","Widow","WWWOFFLE","Xaldon WebSpider","Zeus"]}
{"domain":"cygwin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["Scooter","Test Crawler","DigExt","CW Crawler","Sosospider","Googlebot","*"]}
{"domain":"ladsp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tranplanet.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"eqtv.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"adentifi.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"awsdns-52.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"toasttab.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["ia_archiver","archive.org_bot","*"]}
{"domain":"adidas.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"transip.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hurriyet.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":14,"stanza_tokens":["*","Flipboard","FlipboardProxy","GPTBot","anthropic-ai","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","OAI-SearchBot","quillbot.com","PerplexityBot","TurnitinBot","AmazonAdBot"]}
{"domain":"sankuai.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"santander.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"malwarebytes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Googlebot","applebot","PerplexityBot","bingbot","OAI-SearchBot","GPTBot","GeminiBot","ClaudeBot"]}
{"domain":"snapcraft.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"vt.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["bingbot","SemrushBot","meta-externalagent","facebookexternalhit","*"]}
{"domain":"docusign.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Twitterbot"]}
{"domain":"mossfast.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.com.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"wikibooks.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["MJ12bot","Mediapartners-Google*","IsraBot","Orthogaffe","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","fast","wget","grub-client","k2spider","NPBot","WebReaper","SemrushBot","*"]}
{"domain":"thingiverse.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Googlebot","Bingbot","DuckDuckBot","Yandex","SemrushBot"]}
{"domain":"wowhead.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":31,"stanza_tokens":["AI2Bot","Ai2Bot-Dolma","anthropic-ai","Applebot-Extended","Bytespider","CCBot","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","Meta-ExternalAgent","Meta-ExternalFetcher","omgili","omgilibot","Scrapy","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended","Mediapartners-Google","*"]}
{"domain":"amx1.net","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":[400,404]}
{"domain":"savefrom.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["googlebot","*"]}
{"domain":"dvr163.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"gmarket.co.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","GoogleBot","BingBot","Yeti","Daumoa","Twitterbot","GPTBot","AdsBot-Google","Ads-Naver","GoogleOther","facebookexternalhit","facebookcatalog"]}
{"domain":"minutemedia-prebid.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"viasat.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zscaler.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"com.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"threatlocker.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"usp.br","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"tradingeconomics.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"symcb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"siteimproveanalytics.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rtve.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":148,"stanza_tokens":["*","Googlebot","Googlebot-News","Googlebot-Image","Googlebot-Video","008","Alexibot","Aqua_Products","b2w/0.1","BackDoorBot/1.0","Bookmark search tool","BotALot","BotRightHere","BuiltBotTough","Bullseye/1.0","BunnySlippers","CherryPicker","CherryPickerSE/1.0","CherryPickerElite/1.0","CheeseBot","CNCDialer","Copernic","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","DittoSpyder","DOC","Download Ninja","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","ExtractorPro","FairAd Client","fast","Fetch","Flaming AttackBot","Foobot","Gaisbot","GetRight/4.2","grub-client","Harvest/1.5","hloader","httplib","HTTrack","HTTrack 3.0","humanlinks","ImagesiftBot","InfoNaviRobot","Iron33/1.0.2","JennyBot","JikeSpider","Jyxobot/1","k2spider","Kenjin Spider","Keyword Density/0.9","larbin","LexiBot","libWeb/clsHTTP","libwww","LinkextractorPro","linko","LinkScan/8.1a Unix","LinkWalker","LNSpiderguy","lwp-trivial/1.34","lwp-trivial","Mata Hari","Maxthon","MIIxpc","MIIxpc/4.2","Mister PiX","moget","moget/2.1","NetAnts","NICErsPRO","NPBot","Offline Explorer","Openbot","Openfind data gatherer","Openfind","Oracle Ultra Search","PerMan","ProWebWalker","ProPowerBot/2.14","Python-urllib","QueryN Metasearch","Radiation Retriever 1.1","RepoMonkey Bait & Tackle/v1.01","RepoMonkey","RMA","searchpreview","sitecheck.internetseer.com","SiteSnagger","SpankBot","spanner","suzuran","Szukacz/1.4","Teleport","TeleportPro"]}
{"domain":"zara.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"softonic.com","fetch_ok":false,"usable":false,"reason":"other","status":412,"error":null,"attempts":[412,412]}
{"domain":"bigo.sg","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"greenpeace.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unep.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pp.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"DEPTH_ZERO_SELF_SIGNED_CERT"]}
{"domain":"gitee.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"scopus.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"manulife-sinochem.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"adingo.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ml314.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ECONNRESET"]}
{"domain":"chartbeat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"permutive.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","AdsBot-Google","AdsBot-Google-Mobile","GoogleOther"]}
{"domain":"monday.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["ChatGPT-User","OAI-SearchBot","PerplexityBot","Perplexity-User","ClaudeBot","GoogleOther","Google-Extended","*","Skynet"]}
{"domain":"wireshark.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"consumerreports.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"georgetown.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.co.il","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"mcdonalds.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["SiteAuditBot","Spacecat/1.0","*"]}
{"domain":"eventbrite.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":139,"stanza_tokens":["*","Facebot","Slurp","CCBot","AhrefsBot","Balihoo","BotRightHere","WebZip","larbin","b2w/0.1","Copernic","psbot","Python-urllib","NetMechanic","URL_Spider_Pro","CherryPicker","EmailCollector","EmailSiphon","WebBandit","EmailWolf","ExtractorPro","CopyRightCheck","Crescent","SiteSnagger","ProWebWalker","CheeseBot","LNSpiderguy","Alexibot","Teleport","TeleportPro","MIIxpc","Telesoft","Website Quester","WebZip/4.0","WebStripper","WebSauger","WebCopier","NetAnts","Mister PiX","WebAuto","TheNomad","WWW-Collector-E","RMA","libWeb/clsHTTP","asterias","httplib","turingos","spanner","InfoNaviRobot","Harvest/1.5","Bullseye/1.0","Mozilla/4.0 (compatible; BullsEye; Windows 95)","Crescent Internet ToolPak HTTP OLE Control v.1.0","CherryPickerSE/1.0","CherryPickerElite/1.0","WebBandit/3.50","NICErsPRO","Microsoft URL Control - 5.01.4511","DittoSpyder","Foobot","SpankBot","BotALot","lwp-trivial/1.34","lwp-trivial","BunnySlippers","Microsoft URL Control - 6.00.8169","URLy Warning","Wget/1.6","Wget/1.5.3","Wget","LinkWalker","cosmos","moget","hloader","humanlinks","LinkextractorPro","Offline Explorer","Mata Hari","LexiBot","Web Image Collector","The Intraformant","True_Robot/1.0","True_Robot","BlowFish/1.0","JennyBot","MIIxpc/4.2","BuiltBotTough","ProPowerBot/2.14","BackDoorBot/1.0","toCrawl/UrlDispatcher","WebEnhancer","suzuran","TightTwatBot","VCI WebViewer VCI WebViewer Win32","VCI","Szukacz/1.4","QueryN Metasearch","Openfind data gatherer","Openfind","Xenu's Link Sleuth 1.1c"]}
{"domain":"hoster.kz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wustl.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sjv.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ftstatic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nextmillmedia.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mathworks.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"zscalertwo.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"samplicio.us","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"dns-finder.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"isc.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"videolan.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":418,"error":null,"attempts":[418,418]}
{"domain":"163img.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"ctnsnet.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"rackspaceclouddb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"deadline.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":26,"stanza_tokens":["*","anthropic-ai","cohere-ai","omgili","omgilibot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ChatGPT","DeepSeek","DeepSeekBot","GPTBot","OAI-SearchBot","ClaudeBot","Claude-Web","Diffbot","FacebookBot","Google-Extended","Meta-ExternalAgent","Meta-ExternalFetcher","PerplexityBot","Timpibot","Webzio-Extended","YouBot","cXensebot"]}
{"domain":"myanimelist.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":47,"stanza_tokens":["*","AI2Bot","Ai2Bot-Dolma","aiHitBot","anthropic-ai","Applebot-Extended","Brightbot 1.0","CCBot","ChatGLM-Spider","ClaudeBot","Claude-Web","cohere-training-data-crawler","Cotoyogi","DeepSeekBot","Diffbot","FacebookBot","facebookexternalhit","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Google-Extended","GPTBot","ICC-Crawler","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","laion-huggingface-processor","LAIONDownloader","Linguee Bot","Meta-ExternalAgent","meta-externalagent","omgili","omgilibot","PanguBot","SBIntuitionsBot","Scrapy","Sidetrade indexer bot","Spider","TerraCotta","TikTokSpider","Timpibot","VelenPublicWebCrawler","Webzio-Extended","webzio-extended","YandexAdditional","YandexAdditionalBot","Bytespider"]}
{"domain":"rtbhouse.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["DatoCMSSearchBot","*"]}
{"domain":"mediavine.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kasperskylabs.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNRESET"]}
{"domain":"delta.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"iionads.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"ntt.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ENETUNREACH","attempts":["ENETUNREACH","TimeoutError"]}
{"domain":"awsdns-27.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"parrot-flow.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"the-ozone-project.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"siteground.biz","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"dns.br","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"socdm.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"sentry-cdn.com","fetch_ok":false,"usable":false,"reason":"other","status":500,"error":null,"attempts":["ENOTFOUND",500,"TimeoutError"]}
{"domain":"bbci.co.uk","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","TimeoutError"]}
{"domain":"queniuak.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"idealo.de","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":[503,503,"TimeoutError"]}
{"domain":"swiftkey.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":["TimeoutError",429,"TimeoutError"]}
{"domain":"mfms.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"google.co.th","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"webnovel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gog.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"infojobs.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["Mediapartners-Google","AdsBot-Google","AdsBot-Google-Mobile","googlebot","*","LinkedInBot","LinkedInBot/1.0","jobrapido"]}
{"domain":"lavanguardia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":37,"stanza_tokens":["*","ia_archiver","UbiCrawler","YandexBot/3.0","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","Slurp","Maxthon","CNCDialer","AhrefsBot","mailto: techinfo@ubermetrics-technologies.com","GPTBot","anthropic-ai","Claude-Web","ClaudeBot","YouBot","Bytespider","CCBot"]}
{"domain":"axis.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"federalregister.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"freedesktop.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"streamtheworld.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"shuashuati.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot"],"open":[],"path_restricted":["Claude-User","anthropic-ai"]}],"stanza_token_count":25,"stanza_tokens":["*","SemrushBot","ThinkChaos","LinkpadBot","amazonbot","gptbot","yandex","petalbot","dotbot","semrush","ClaudeBot","AwarioSmartBot","DuckDuckBot","INETDEX-BOT","DataForSeoBot","ImagesiftBot","DotBot","oBot","SeznamBot","YandexBot","iaskspider","SkyworkSpider","meta-externalagent","Claude-SearchBot","AhrefsBot"]}
{"domain":"iinet.net.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hrw.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["AhrefsBot","SemrushBot","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SemrushBot-CT","SemrushBot-BM","SemrushBot-COUB","SiteAuditBot","SplitSignalBot","LinkCheck by Siteimprove.com","*"]}
{"domain":"mtrtb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AdsBot-Google-Mobile","AdsBot-Google","PetalBot","dotbot","AhrefsBot"]}
{"domain":"ondemand.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Nuclei*","*jndi*","*"]}
{"domain":"t2.ru","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":[503,"TimeoutError",503]}
{"domain":"pure.cloud","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"logitech.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vanityfair.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","FacebookExternalHit","Twitterbot","Pinterestbot","LinkedInBot","TikTokSpider","Storebot-Google","Google-InspectionTool","AmazonAdBot","Google-Extended","Google-CloudVertexBot","GoogleOther","Applebot-Extended","Amazonbot","meta-webindexer","meta-externalagent","meta-externalfetcher","ClaudeBot","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","cohere-training-data-crawler","cohere-ai","CCBot","PanguBot","PetalBot","Bytespider","Diffbot","DuckAssistBot","MistralAI-User","Timpibot","Webzio","Webzio-Extended","archive.org_bot","ia_archiver","ia_archiver-web.archive.org","heritrix"]}
{"domain":"msi.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"parastorage.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ECONNRESET"]}
{"domain":"thetimes.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"nzherald.co.nz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":319,"stanza_tokens":["BingBot","msnbot","Slurp","Yahoo-Newscrawler","Googlebot","AASA-Bot/1.0.0","AASA-Bot","Apple-Bot/1.0.0","Apple-Bot","Googlebot-News","EchoboxBot/1.0","*","anthropic-ai","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","GPTBot","ChatGPT-User","OAI-SearchBot","cohere-ai","PerplexityBot","Perplexity-User","Meta-ExternalAgent","meta-externalagent","Meta-ExternalFetcher","meta-externalfetcher","Google-Extended","Google-CloudVertexBot","Applebot-Extended","AliyunSecBot","Bytespider","CCBot","quillbot.com","DuckAssistBot","MeltwaterNews","psbot","Yahoo-MMCrawler","grapeshot","*OrkashBot*","*Flamingo_SearchEngine*","*BLP_bbot*","*Birubot*","*PaperLiBot*","*aiHitBot*","*SiteBot*","*yacybot*","*MLBot*","*Yeti*","*ExaBot*","*NaverBot*","*sindice-fetcher*","*MJ12bot*","*DotBot*","*trendictionbot*","*Ezooms*","*YandexBot*","*Fetch*","*libwww*","*newsexpress*","aipbot","aipbot*","aipbot/1.0","Alexibot","Aqua_Products","archive.org_bot","ArchiveTeam","BackDoorBot","Bilbo/2.3b-UNIX","Black Hole","BlowFish","BlowFish/1.0","BorderManager*","BotRightHere","BuiltBotTough","Bullseye","Bullseye/1.0","BunnySlippers","CherryPickerElite/1.0","Copernic","CopyRightCheck","Crescent Internet ToolPak HTTP OLE Control v.1.0","Crescent Internet ToolPak HTTPOLE Control v.1.0","DISCo Pump 3.2","DISCoFinder","Download Demon/3.2.0.8","dumbot","eCatch/3.0","EmailWolf","EroCrawler","ExtractorPro","FAST-WebCrawler/3.6/FirstPage","FlashGet","FlashGet WebWasher 3.2","FrontPage [NC,OR]","GetRight/3.1","GetRight/3.3","GetRight/4.1.0","GetRight/4.2","GetRight/4.2b (Portuguxeas)"]}
{"domain":"tynt.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"es.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hopkinsmedicine.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wixstatic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"xero.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tvinteractive.tv","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"apple-dns.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"samsclub.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wistia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nicovideo.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":3,"stanza_tokens":["Googlebot","GPTBot","*"]}
{"domain":"iterable.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"digitaltrends.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":26,"stanza_tokens":["*","omgili","diffbot","kangaroo","img2dataset","Greppr","TikTokSpider","YandexBot","Yandex","SeznamBot","PetalBot","Bingbot","SemrushBot","youbot","amazonbot","meta-externalagent","meta-webindexer","Baiduspider","Baiduspider-image","Baiduspider-video","Baiduspider-news","Baiduspider-favo","AhrefsBot","OAI-SearchBot","facebookexternalhit","Googlebot"]}
{"domain":"awsdns-59.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"amazon.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"proof-dns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"shazam.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"handelsblatt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User"]}],"stanza_token_count":21,"stanza_tokens":["*","GPTBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","anthropic-ai","ClaudeBot","Claude-Web","Claude-SearchBot","Diffbot","Meta-ExternalAgent","FacebookBot","YouBot","Timpibot","Amazonbot","DeepSeek","DeepSeekBot","cohere-ai","MistralAI-User","Cloudflare-Crawl"]}
{"domain":"webgo.de","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"tufts.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"timesofisrael.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"gitbook.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"nobelprize.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dnsv1.com.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"onlinepbx.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-30.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"blogspot.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"speedifyvolcai.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"shawcable.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"macys.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vervegroupinc.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",404]}
{"domain":"rs6.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mimecast.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zdf.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":21,"stanza_tokens":["CCBot","Google-Extended","GPTBot","ChatGPT-User","scrapy","scrapybot","SemrushBot","sentibot","SEOkicks-Robot","TkBot","trendkite-akashic-crawler","Vagabondo","WBSearchBot","grapeshot","iCjobs","linkdexbot","magpie-crawler","MegaIndex","MJ12bot","proximic","QuerySeekerSpider"]}
{"domain":"vatican.va","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"claude.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zohocdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tencentclb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bidvol.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"teads.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"apptentive.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tripadvisor.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":25,"stanza_tokens":["Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Cohere-ai","GPTBot","Google-CloudVertexBot","Google-Extended","Img2dataset","PanguBot","PetalBot","SentiBot","YouBot","magpie-crawler","meta-externalagent","*","ChatGPT-User","ChatGPT-User/2.0","Gemini-Deep-Research","OAI-SearchBot","applebot","PerplexityBot","bingbot","Yandex"]}
{"domain":"signalr.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"goguardian.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"joyn.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"trouble-free.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"wikidot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["voltron"]}
{"domain":"gotinder.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"riskified.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sas.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":30,"stanza_tokens":["Googlebot","Bingbot","Duckduckbot","Slurp","Baiduspider","YandexBot","AhrefsBot","SemrushBot","Moz","MJ12Bot","facebookexternalhit","Twitterbot","LinkedInBot","Slackbot","Pinterestbot","GPTBot","CCBot","PerplexityBot","Bytespider","Amazonbot","Applebot","SiteimproveBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Perplexity-User","Google-Extended","*"]}
{"domain":"telstra.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":32,"stanza_tokens":["Alexa","Amazon-Kendra","AmazonBot","ByteDance","ByteSpider","CC-Crawler","CCBot","ChatGLM","ClearScopeBot","CommonCrawl","DataForSEOBot","Diffbot","Falcon","FlyRiver","Goose","Grammarly","Iaskspider","imagesiftBot","Img2dataset","PetalBot","Quillbot","Scrapy","Sidetrade","Sonic","TikTokSpider","Timpibot","VelenPublicWebCrawler","Swiftbot","Elastic-Crawler Elastic Cloud","Elastic-Crawler","Elastic Cloud","*"]}
{"domain":"yle.fi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":39,"stanza_tokens":["*","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Crawlspace","DeepSeek","DeepSeekBot","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","Grok","GrokAI","iaskspider/2.0","ICC-Crawler","img2dataset","ISSCyberRiskCrawler","Meta-ExternalAgent","omgili","omgilibot","PanguBot","PetalBot","Scrapy","Sidetrade indexer bot","Timpibot","TurnitinBot","VelenPublicWebCrawler","x.AI","xAI","YouBot"]}
{"domain":"ripe.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"advolve.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"sprinthost.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","*"]}
{"domain":"houzz.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"adotmob.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"awsdns-39.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"found.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"snowflake.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cbssports.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":2,"stanza_tokens":["*","GPTBot"]}
{"domain":"smartsheet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"weatherbug.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity‑User"]}
{"domain":"samsunghealth.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"resume.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"voximplant.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ipcc.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nic.zone","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"paa-reporting-advertising.amazon","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"samsungpositioning.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"4shared.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"feedly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"freenet.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"asos.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"pointtoserver.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"emb-api.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"postman.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"takeaway.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"habr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Yandex","Googlebot","Slurp","*"]}
{"domain":"glance-cdn.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"dingtalk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["Googlebot","AdsBot-Google","Googlebot-Image","Baiduspider","CCBot","GPTBot","ClaudeBot","ByteBot","*"]}
{"domain":"pub.network","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"keplr.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"honor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"chron.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","anthropic-ai"],"open":[],"path_restricted":["Claude-SearchBot"]}],"stanza_token_count":24,"stanza_tokens":["ai2bot-dolma","anthropic-ai","applebot-extended","bytespider","ccbot","claude-user","claude-web","claudebot","cohere-ai","diffbot","duckassistbot","friendlycrawler","omgili","omgilibot","perplexity-user","perplexitybot","scrapy","timpibot","youbot","webzio-extended","kangaroo-bot","google-extended","AmazonAdBot","*"]}
{"domain":"flightaware.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":35,"stanza_tokens":["UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","grub-client","k2spider","NPBot","WebReaper","Irvine","360Spider","bingbot","wget","*","Applebot","Cliqzbot","Twitterbot","GPTBot"]}
{"domain":"indiana.edu","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"dhl.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","Googlebot","Bingbot","CCBot","ClaudeBot","PerplexityBot","Amazonbot","Applebot"]}
{"domain":"alchemer.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"heyzine.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mooo.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"nhl.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"monster.es","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE",403]}
{"domain":"bitly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","PetalBot","ChatGPT Agent","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","GPTBot","CCBot","ByteDance"]}
{"domain":"tagesspiegel.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":18,"stanza_tokens":["*","GPTBot","Applebot-Extended","CCBot","CloudVertexBot","ClaudeBot","FacebookBot","meta-externalagent","DeepseekBot","Bytespider","cohere-training-data-crawler","PanguBot","Timpibot","AI2Bot","Diffbot","Omgili","Webzio-Extended","AmazonAdBot"]}
{"domain":"go.dev","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"measureadv.com","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":[400,400]}
{"domain":"olx.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-33.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"braze.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bale.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"servidoresdns.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-42.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"braintree-api.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mobilefuse.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rylkngdm.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"mynavi.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"envato.market","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gatech.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gameloft.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":108,"stanza_tokens":["URL_Spider_Pro","CherryPicker","EmailCollector","EmailSiphon","WebBandit","EmailWolf","ExtractorPro","CopyRightCheck","Crescent","SiteSnagger","ProWebWalker","CheeseBot","LNSpiderguy","Black Hole","Titan","WebStripper","NetMechanic","Wget","mozilla/4","mozilla/5","Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)","Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)","Mozilla/4.0 (compatible; MSIE 4.0; Windows 98)","Teleport","TeleportPro","MIIxpc","Telesoft","Website Quester","WebZip","moget/2.1","WebZip/4.0","WebSauger","WebCopier","NetAnts","Mister PiX","WebAuto","TheNomad","WWW-Collector-E","RMA","libWeb/clsHTTP","asterias","httplib","turingos","spanner","InfoNaviRobot","Harvest/1.5","Bullseye/1.0","Mozilla/4.0 (compatible; BullsEye; Windows 95)","Crescent Internet ToolPak HTTP OLE Control v.1.0","CherryPickerSE/1.0","CherryPickerElite/1.0","WebBandit/3.50","NICErsPRO","Microsoft URL Control - 5.01.4511","DittoSpyder","Foobot","WebmasterWorldForumBot","SpankBot","BotALot","lwp-trivial/1.34","lwp-trivial","Wget/1.6","BunnySlippers","Microsoft URL Control - 6.00.8169","URLy Warning","Wget/1.5.3","LinkWalker","cosmos","moget","hloader","humanlinks","LinkextractorPro","Offline Explorer","Mata Hari","LexiBot","Web Image Collector","The Intraformant","True_Robot/1.0","True_Robot","BlowFish/1.0","JennyBot","MIIxpc/4.2","BuiltBotTough","ProPowerBot/2.14","BackDoorBot/1.0","toCrawl/UrlDispatcher","WebEnhancer","TightTwatBot","suzuran","VCI WebViewer VCI WebViewer Win32","VCI","Szukacz/1.4","QueryN Metasearch","Openfind data gathere","Openfind","Xenu's Link Sleuth 1.1c","Xenu's","Zeus","RepoMonkey Bait & Tackle/v1.01","RepoMonkey"]}
{"domain":"ec-lyon.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adhigh.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UND_ERR_SOCKET","attempts":["UND_ERR_SOCKET","UND_ERR_SOCKET"]}
{"domain":"mrf.io","fetch_ok":false,"usable":false,"reason":"other","status":500,"error":null,"attempts":[500,"ENOTFOUND",500]}
{"domain":"spglobal.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"edx.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"seekingalpha.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":[],"path_restricted":["GPTBot","OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User"],"open":[],"path_restricted":["Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":110,"stanza_tokens":["*","AddSearchBot","AI2Bot","Ai2Bot-Dolma","aiHitBot","AliyunSecBot","Amazonbot","Andibot","Applebot-Extended","ArchiveBot","Arquivo-web-crawler","AudigentAdBot","Awario","AwarioRssBot","AwarioSmartBot","bedrockbot","bigsur.ai","BLEXBot","BLP_bbot/0.1","Brightbot 1.0","BrightEdge","Bytespider","CCBot","ChatGPT-User","Claude-User","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","DataForSeoBot","Datenbank Crawler","Devin","Diffbot","Discordbot","dotbot","DuckAssistBot","Echobot Bot","EchoboxBot","europarchive.org","FacebookBot","Factset_spyderbot","Feedly","FirecrawlAgent","FriendlyCrawler","Google-CloudVertexBot","GoogleAgent-Mariner","heritrix","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Jetslide","LinkArchiver","LinerBot","magpie-crawler","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","MistralAI-User","MistralAI-User/1.0","MJ12bot","Moreover","MyCentralAIScraperBot","netEstate Imprint Crawler","NewsNow","news-please","Nicecrawler","NovaAct","omgili","omgilibot","Operator","PanguBot","Panscient","panscient.com","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Perplexity-User","Perplexity‑User","PhindBot","Poseidon Research Crawler","proximic","QualifiedBot","QuillBot","quillbot.com","Quora-Bot","SBIntuitionsBot","Scrapy","Screaming Frog SEO Spider","SeekrBot","Sidetrade indexer bot","SiteimproveBot","TaraGroup Intelligent Bot","TelegramBot","Thinkbot","TikTokSpider"]}
{"domain":"amazon.ae","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"ouraring.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eum-appdynamics.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"org.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"plex.bz","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"salon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":66,"stanza_tokens":["*","AhrefsBot","SemrushBot","MJ12bot","DotBot","Zoominfobot","BlexBot","Cliqzbot","DataForSeoBot","MauiBot","MegaIndex.ru","Pixray-Seeker","Copyleaks","Plaghunter","ImageMax","Tineye","Copytrack","CopytrackBot","Googlebot","Bingbot","DuckDuckBot","Baiduspider","YandexBot","Sogou","PetalBot","AspiegelBot","SeznamBot","Bytespider","VoilaBot","Twitterbot","FacebookExternalHit","LinkedInBot","Pinterestbot","TelegramBot","Discordbot","WhatsApp","Slackbot","Redditbot","Verity/1.1 (https://gumgum.com/verity; verity-support@gumgum.com)","Scope3/2.0 (scope3.com)","AdsBot-Google","AdsBot-Google-Mobile","Googlebot-Image","Googlebot-Video","Googlebot-News","Googlebot-InspectionTool","GPTBot","ChatGPT-User","ChatGPT-Operator","OAI-SearchBot","ClaudeBot","Claude-User","Claude-SearchBot","Claude-Web","anthropic-ai","PerplexityBot","Perplexity-User","CCBot","Google-Extended","Applebot","Applebot-Extended","Amazonbot","Meta-ExternalAgent","Meta-ExternalFetcher","Facebookbot","AmazonAdBot"]}
{"domain":"awsdns-24.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"foxbusiness.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"boston.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":25,"stanza_tokens":["*","AmazonAdBot","AI2Bot-Dolma","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot-Extended","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","Diffbot","FacebookBot","FirecrawlAgent","Google-Extended","GoogleOther","GPTBot","imageSpider","Meta-ExternalAgent","omgili","PanguBot","Spider","Timpibot","Webzio-Extended"]}
{"domain":"casasbahia.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"awsdns-20.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"syncingbridge.com","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":[400,400]}
{"domain":"standardnotes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"domaintools.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"nalog.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"thefreedictionary.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":33,"stanza_tokens":["Mediapartners-Google","*","meta-externalagent","GPTBot","OAI-SearchBot","ClaudeBot","Claude-SearchBot","Claude-User","ChatGPT-User","CCBot","Applebot-Extended","Google-Extended","GoogleOther","Google-CloudVertexBot","Amazonbot","Bytespider","TikTokSpider","DuckAssistBot","PerplexityBot","AhrefsBot","SemrushBot","MJ12bot","DotBot","DataForSeoBot","MauiBot","AlphaSeoBot","BLEXBot","SEOkicks-Robot","Wotbox","AspiegelBot","PetalBot","archive.org_bot","ia_archiver"]}
{"domain":"virustotal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["Baiduspider"]}
{"domain":"nydailynews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":68,"stanza_tokens":["*","AI2Bot","Ai2Bot-Dolma","Amazonbot","Amzn-SearchBot","anthropic-ai","Applebot","Applebot-Extended","Archive.org_bot","Baiduspider","Brightbot 1.0","Bytespider","carbon-umbrella-bot","CCBot","ChatGLM-Spider","ChatGPT-User","ClaudeBot","Claude-Searchbot","Claude-User","Claude-Web","Cohere-training-data-crawler","Cotoyogi","Dalvik","Datenbank Crawler","Diffbot","DocSpotBot","DuckAssistBot","FacebookBot","GnowitNewsbot","Google-Extended","GPTBot","grok-crawler","ia_archiver-web.archive.org","ICC-Crawler","imageSpider","Kangaroo Bot","laion-huggingface-processor","LCC","Mantisbot","Meta-ExternalAgent","MistralAI-User","netEstate Imprint Crawler","OAI-SearchBot","omgili","PanguBot","PerplexityBot","Perplexity-User","PetalBot","SBIntuitionsBot","SirdataBot","Spider","Thinkbot","Timpibot","VelenPublicWebCrawler","Vivaldi","Webzio-extended","YandexBot","YouBot","AmazonAdBot","facebookexternalhit","grapeshot","Mozilla/5.0 (compatible; EchoboxBot/1.0; hash/w4mwnpbXf3MFAbxOkJRw; +http://www.echobox.com)","Twitterbot","Mozilla/5.0 (compatible; NewsRoom.BI/0.1; +http://www.newsroom.bi/bot.html)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36(KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; mrfCompass-Marshall/1.0)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Jukebox/1.0)"]}
{"domain":"stuff.co.nz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":21,"stanza_tokens":["grapeshot","*","Amazonbot","Applebot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Diffbot","FacebookBot","Google-Extended","GPTBot","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","omgili","PerplexityBot","Timpibot","Webzio-Extended","YouBot"]}
{"domain":"abebooks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Googlebot","AdsBot","msnbot","bingbot","Slurp","Mediapartners-Google","rogerbot","ClarityBot"]}
{"domain":"bgdns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"lenovomm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"digital.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"oaiusercontent.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"voanews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AhrefsBot","RavenCrawler"]}
{"domain":"revcontent.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"lse.ac.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Atomic_Email_Hunter","emailsiphon","VERSION_TABLE","*"]}
{"domain":"groww.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["GPTBot","PerplexityBot","bingbot","BingPreview","bingGPT","Googlebot","Google-Extended","Gemini","ClaudeBot","*","Googlebot-Mobile"]}
{"domain":"rajasthan.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","ENOTFOUND"]}
{"domain":"dnsv1.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"awsdns-16.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"baishan-cloud.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ENOTFOUND"]}
{"domain":"ethdisco.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"staticflickr.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pinterest.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"termsfeed.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":93,"stanza_tokens":["AhrefsBot","AhrefsSiteAudit","YepBot","SemrushBot","SemrushBot-SA","Rogerbot","Exabot","barkrowler","MJ12bot","rogerbot","Dotbot","dotbot","DataForSeoBot","ia_archiver","magpie-crawler","MBCrawler","iubenda-radar","Gigabot","BlackWidow","ChinaClaw","Custo","DISCo","Download\\ Demon","eCatch","EirGrabber","EmailSiphon","EmailWolf","Express\\ WebPictures","ExtractorPro","EyeNetIE","FlashGet","GetRight","GetWeb!","Go!Zilla","Go-Ahead-Got-It","GrabNet","Grafula","HMView","HTTrack","Image\\ Stripper","Image\\ Sucker","Indy\\ Library","InterGET","Internet\\ Ninja","JetCar","JOC\\ Web\\ Spider","larbin","LeechFTP","Mass\\ Downloader","MIDown\\ tool","Mister\\ PiX","Navroad","NearSite","NetAnts","NetSpider","Net\\ Vampire","NetZIP","Octopus","Offline\\ Explorer","Offline\\ Navigator","PageGrabber","Papa\\ Foto","pavuk","pcBrowser","RealDownload","ReGet","SiteSnagger","SmartDownload","SuperBot","SuperHTTP","Surfbot","tAkeOut","Teleport\\ Pro","VoidEYE","Web\\ Image\\ Collector","Web\\ Sucker","WebAuto","WebCopier","WebFetch","WebGo\\ IS","WebLeacher","WebReaper","WebSauger","Website\\ eXtractor","Website\\ Quester","WebStripper","WebWhacker","WebZIP","Wget","Widow","WWWOFFLE","Xaldon\\ WebSpider","Zeus"]}
{"domain":"500px.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Googlebot","sitebot","Yeti"]}
{"domain":"porsche.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","EmailCollector","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier"]}
{"domain":"bahn.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot","Bingbot","Slurp"]}
{"domain":"asuscomm.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENETUNREACH"]}
{"domain":"urbandictionary.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"hpdaas.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ECONNRESET",200]}
{"domain":"politico.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":94,"stanza_tokens":["*","Googlebot-News","AddSearchBot","AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","Andibot","anthropic-ai","Applebot","Applebot-Extended","Awario","bedrockbot","bigsur.ai","Brightbot 1.0","Bytespider","CCBot","ChatGPT Agent","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Datenbank Crawler","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","FacebookBot","facebookexternalhit","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","LinerBot","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NovaAct","OAI-SearchBot","omgili","omgilibot","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","Thinkbot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","wpbot","YaK","YandexAdditional","YandexAdditionalBot","YouBot"]}
{"domain":"pushwoosh.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","GoogleBot"]}
{"domain":"hs-scripts.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"standard.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AmazonAdBot","Nutch"]}
{"domain":"elementor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nianticlabs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"qnap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AhrefsBot","AhrefsSiteAudit"]}
{"domain":"marphezis.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ksord.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"code.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"orf.at","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":21,"stanza_tokens":["*","stress-agent","fast","Scooter","AhrefsBot","magpie-crawler","CCBot","GPTBot","FacebookBot","omgili","omgilibot","anthropic-ai","Claude-Web","ClaudeBot","cohere-ai","Google-Extended","Bytespider","PetalBot","Diffbot","PerplexityBot","YouBot"]}
{"domain":"1984.is","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-43.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"auth0edge.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"abc.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot"],"open":[],"path_restricted":["Claude-User","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":17,"stanza_tokens":["*","008","Twitterbot","ChatGPT-User","GPTBot","Google-Extended","PerplexityBot","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","OAI-SearchBot","ia_archiver","archive.org_bot","ia_archiver-web.archive.org","Claude-SearchBot"]}
{"domain":"undertone.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"sberdevices.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"doubao.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"printercloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"iqm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jp.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"topdns.me","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-54.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hotels.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cvent.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","Wget","Wget/1.9","Wget/1.6","Wget/1.5.3","AhrefsBot","AhrefsSiteAudit","LinkedInBot"]}
{"domain":"shopifycdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"xtracloud.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"qcomgeo2.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gimp.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"honeywell.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.co.za","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"amnesty.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","CCBot","bingbot"]}
{"domain":"bstatic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["psbot","TurnitinBot","NPBot-1/2.0","NPBot","Applebot","*"]}
{"domain":"artstation.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"freecash.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dreamstime.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["meta-externalagent","*"]}
{"domain":"awsdns-13.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"globalnews.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"gannett-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"onegg.site","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ufileos.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"domain.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vercel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ourworldindata.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"slack-edge.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"lonelyplanet.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"rezync.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"proquest.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":["OAI-SearchBot"],"path_restricted":[]}],"stanza_token_count":4,"stanza_tokens":["*","GPTBot","ChatGPT-User","OAI-SearchBot"]}
{"domain":"sedoparking.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["Googlebot","Mediapartners-Google","Yahoo! Slurp","bingbot","Yandex","Baiduspider","Sogou","ia_archiver","IPS-Agent","BLEXBot","Pandalytics","IonCrawl","*"]}
{"domain":"bounceexchange.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"parsely.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unl.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bi.zone","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","Googlebot"]}
{"domain":"ke.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":22,"stanza_tokens":["Baiduspider","Sogou inst spider","Sogou spider","Googlebot","360Spider","haosouspider","bingbot","msnbot","msnbot-media","Sosospider","Sogou News Spider","Yahoo! Slurp China","Yahoo!","YoudaoBot","YodaoBot","Sogou web spider","YisouSpider","ia_archiver","EasouSpider","JikeSpider","EtaoSpider","*"]}
{"domain":"wscvip.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"365scores.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"splunkdev.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pokemon.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"fox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hepsiburada.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"qpic.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"nfl.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Mediapartners-Google"]}
{"domain":"schneevonmorgen.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"is.gd","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"awsdns-36.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-49.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"googlehosted.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"google.com.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"smallpdf.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"trkn.us","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"hs-analytics.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rochester.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["atraxbot/0.2","*","ReadableBot"]}
{"domain":"hawaii.edu","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"mhlw.go.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"timeout.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","80bot","genieBot","MJ12bot","MLBot","Nutch","OmniExplorer_Bot","spock","BecomeBot","TurnitinBot"]}
{"domain":"vtex.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"arduino.cc","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sbb.ch","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"awsdns-12.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gemius.pl","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ad-score.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["GoogleOther","*"]}
{"domain":"br.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":21,"stanza_tokens":["*","applebot-extended","bytespider","ccbot","claudebot","google-extended","cohere-training-data-crawler","deepseekbot","deepseek","deepseek-crawler","diffbot","facebookbot","meta-externalagent","gptbot","grokbot","img2dataset","laiondownloader","mistralai-crawler","omgili","omgilibot","pangubot"]}
{"domain":"cpi.ad.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"quark.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"meoodata01.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wscdns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"citi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"krushmedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"us.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"insightexpressai.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-41.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-35.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"walmartimages.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-26.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"jivosite.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"reverso.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"brightcloud.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"prserv.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cia.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["usasearch","Mediapartners-Google*","bingbot","msnbot","IsraBot","Orthogaffe","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","wget","grub-client","k2spider","NPBot","WebReaper","*"]}
{"domain":"google.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"sovcombank.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":630,"stanza_tokens":["Yandex","Googlebot","01h4x.com","360Spider","404checker","404enemy","80legs","Abonti","Aboundex","Aboundexbot","Acunetix","ADmantX","adscanner","AdsTxtCrawlerTP","AfD-Verbotsverfahren","AIBOT","AiHitBot","Aipbot","Alexibot","ALittle Client","Alligator","AllSubmitter","AlphaBot","Anarchie","Anarchy","Anarchy99","Ankit","Anthill","Apexoo","arquivo-web-crawler","arquivo.pt","Aspiegel","ASPSeek","Asterias","Atomseobot","Attach","autoemailspider","awario.com","AwarioBot","AwarioRssBot","AwarioSmartBot","BackDoorBot","Backlink-Ceck","backlink-check","BacklinkCrawler","BackStreet","BackWeb","Badass","Bandit","Barkrowler","BatchFTP","Battleztar Bazinga","BBBike","BDCbot","BDFetch","BetaBot","Bigfoot","Bitacle","Black Hole","Blackboard","BlackWidow","BLEXBot","Blow","BlowFish","Boardreader","Bolt","BotALot","Brandprotect","Brandwatch","Buck","Buddy","BuiltBotTough","BuiltWith","Bullseye","BunnySlippers","BuzzSumo","Bytespider","cah.io.community","Calculon","CATExplorador","CazoodleBot","CCBot","Cegbfeieh","CensysInspect","check1.exe","CheeseBot","CherryPicker","CheTeam","ChinaClaw","Chlooe","Citoid","Claritybot","clark-crawler","Cliqzbot","Cloud mapping","coccocbot","Cocolyzebot","CODE87","Cogentbot","cognitiveseo"]}
{"domain":"hackaday.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"e-recht24.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","newspaper/0.2.8","wget","webzip","webmirror","webcopy","magpie-crawler"]}
{"domain":"colorlib.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"qunar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"samsungcloudcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rima-tde.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"lnk.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-08.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"oppomobile.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"sa.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"thestreet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ndr.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":21,"stanza_tokens":["*","applebot-extended","bytespider","ccbot","claudebot","google-extended","cohere-training-data-crawler","deepseekbot","deepseek","deepseek-crawler","diffbot","facebookbot","meta-externalagent","gptbot","grokbot","img2dataset","laiondownloader","mistralai-crawler","omgili","omgilibot","pangubot"]}
{"domain":"vodafone-ip.de","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"mariadb.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"banki.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"lua.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"awsdns-47.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-25.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"glpals.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"buzzsprout.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.co.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"avg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","Yandex","Active Cache Request","Googlebot-image","OAI-SearchBot","ChatGPT-User","GPTBot","Perplexity-User","PerplexityBot","Claude-User","ClaudeBot"]}
{"domain":"aeroflot.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"macomnet.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"xhtotal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"xx1t.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ttdns3.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"radiofrance.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":65,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","CCBot","anthropic-ai","Claude-Web","Applebot-Extended","Amazonbot","Bytespider","ClaudeBot","cohere-ai","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","PerplexityBot","omgili","omgilibot","magpie-crawler","ProRataInc","PetalBot","cohere-training-data-crawler","Scrapy","img2dataset","DigitalOceanGenAICrawler","MistralAI-User","MistralAI-User/1.0","DuckAssistBot","Grok","GrokBot","DeepSeekBot","PanguBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","amazon-kendra","bedrockbot","Channel3Bot","ChatGLM-Spider","Cotoyogi","EchoboxBot","Echobot Bot","Gemini-Deep-Research","Google-CloudVertexBot","Google-NotebookLM","GoogleAgent-Mariner","iAskBot","iaskspider","iaskspider/2.0","ICC-Crawler","ImagesiftBot","KunatoCrawler","LinerBot","meta-webindexer","QualifiedBot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","wpbot","WRTNBot","ZanistaBot","KlaviyoAIBot"]}
{"domain":"legislation.gov.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lenovo.com.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["lrbannerspider","*"]}
{"domain":"brown.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"space.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","AmazonAdBot","bytespider","mistralai","cohere","ai2bot","youbot","omgili","diffbot","kangaroo","img2dataset","amazonbot","amazon-qbusiness","meta-externalagent","meta-webindexer"]}
{"domain":"adjoe.zone","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":[503,"ENOTFOUND",503]}
{"domain":"dataprivacyframework.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xkcd.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":2,"stanza_tokens":["*","GPTBot"]}
{"domain":"dns.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Baiduspider"]}
{"domain":"dashlane.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":25,"stanza_tokens":["*","europarchive.org","GPTBot","ChatGPT-User","OAI-SearchBot","OpenAI-Operator","ClaudeBot","Claude-Web","anthropic-ai","Google-Extended","PerplexityBot","Applebot","FacebookBot","Meta-ExternalAgent","BingSearch","Grok","cohere-ai","Amazonbot","DeepSeek","DeepSeek Assistant","R1","YouBot","CCBot","Googlebot","Bingbot"]}
{"domain":"fraunhofer.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rzd.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bac.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","TimeoutError"]}
{"domain":"awsdns-34.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dns.it","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"symcd.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"home.blog","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"over-blog.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"10jqka.com.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hadronid.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sba.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"man7.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"changeip.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"uwaterloo.ca","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ck-ie.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"clinicaltrials.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-10.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vercel-dns-013.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"adscale.de","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"awsdns-22.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"monash.edu","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"osha.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pendingrenewaldeletion.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"freshchat.com","fetch_ok":false,"usable":false,"reason":"other","status":502,"error":null,"attempts":[502,502,502]}
{"domain":"ioriveredge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"myftpupload.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"selcloud.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"smugmug.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":39,"stanza_tokens":["Adsbot-Google","Alexabot","BingPreview","CloudflarePrefetch","FriendFeedBot","FunnelBack","Google","Google Favicon","Google-Site-Verification","Google-Sitemaps","Googlebot","Googlebot-Image","Googlebot-Mobile","Googlebot-News","Googlebot-Video","Mediapartners-Google","Pingdom","Pinterest","ScoutJet","Slurp","Spinn3r","Teoma","Twitterbot","Yandex","YandexImages","YandexVideoParser","Yeti","archive.org_bot","baiduspider","bingbot","facebookexternalhit","gsa-crawler","houzzbot","ia_archiver","msnbot","rogerbot","zombies","werewolves","*"]}
{"domain":"ekatox.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"scene7.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"wasabisys.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"bk6bba-resources.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"garant.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","YandexDirect","Yandex"]}
{"domain":"indiatoday.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot-News","Mediapartners-Google"]}
{"domain":"google.com.sg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"awsdns-07.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"bitrix24.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":26,"stanza_tokens":["*","Bingbot","MSNBot","DuckDuckBot","Baiduspider","Slurp","Googlebot","Googlebot-Image","Googlebot-Mobile","Googlebot-News","Googlebot-Video","Yandexbot","GPTBot","ChatGPT-User","anthropic-ai","Claude-Web","ClaudeBot","Google-Extended","CCBot","cohere-ai","Perplexity-bot","Bingbot-Copilot","OAI-SearchBot","Facebookbot","YouBot","DuckAssistBot"]}
{"domain":"ziggo.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["facebookexternalhit","*"]}
{"domain":"celtra.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nic.video","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"emarsys.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oray.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-62.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"confiant-integrations.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.com.ph","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"t0x.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"gnome.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"wdr.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":22,"stanza_tokens":["applebot-extended","bytespider","ccbot","claudebot","google-extended","cohere-training-data-crawler","deepseekbot","deepseek","deepseek-crawler","diffbot","facebookbot","meta-externalagent","gptbot","grokbot","img2dataset","laiondownloader","mistralai-crawler","omgili","omgilibot","pangubot","ICCrawler - iCjobs","*"]}
{"domain":"awsdns-51.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"htzq.com.cn","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"cloudappsecurity.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"runescape.wiki","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":[],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot"],"open":[],"path_restricted":["Claude-User","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":38,"stanza_tokens":["*","AhrefsBot","DotBot","Mauibot","MJ12bot","PetalBot","SemrushBot","DataForSeoBot","Neevabot","MegaIndex","Seekport","Barkrowler","ImagesiftBot","BLEXBot","SenutoBot","GPTBot","CCBot","Google-Extended","yacybot","meta-externalagent","meta-webindexer","ClaudeBot","Claude-SearchBot","AI2Bot","OAI-SearchBot","thetradedesk","carbon-umbrella-bot","PerplexityBot","Bytespider","TerraCotta","SiteSucker","Applebot","Brightbot","Aranet","SleepBot","SERankingBacklinksBot","AmazonProductDiscoveryBot","Dragonfly"]}
{"domain":"volc-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"prtimes.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unity3dusercontent.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"softether-network.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"stickyadstv.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"jpost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["Googlebot","Twitterbot","GPTBot","ChatGPT-User","OAI-SearchBot","Claude-User","Perplexity-User"]}
{"domain":"tesco.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"qualcomm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["daumoa","Ultraseek","Ultraseek-www","ia_archiver","twiceler","SEOkicks-Robot"]}
{"domain":"sng.link","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"legacy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Mediapartners-Google","Opebot-v (https://www.1plusx.com)","AmazonAdBot","*"]}
{"domain":"lazada.co.id","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot-Image"]}
{"domain":"telstra.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"google.com.my","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"pcworld.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":37,"stanza_tokens":["*","Ai2Bot-Dolma","AI2Bot","anthropic-ai","Applebot-Extended","Arquivo-web-crawler","AwarioRssBot","AwarioSmartBot","Barkrowler","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","crawler4j","dotbot","Diffbot","FacebookBot","FriendlyCrawler","GPTBot","https://hada.news","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","MJ12bot","PanguBot","Scrapy","Seekr","Timpibot","VelenPublicWebCrawler","Webzio-Extended"]}
{"domain":"gorgias.chat","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bytelb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hbs.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Bingbot","*"]}
{"domain":"chosun.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":3,"stanza_tokens":["*","GPTBot","DeepSeekBot"]}
{"domain":"stspg-customer.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"zscalerthree.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"faceit.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"unam.mx","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE",403]}
{"domain":"brightcove.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bb.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mtproto.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iastate.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"metmuseum.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":[429,429,429]}
{"domain":"morningstar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gcd-dns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"17track.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ansa.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":21,"stanza_tokens":["*","Claudebot","ChatGPT-User","ImagesiftBot","Seekr","CCBot","Claude-Web","PerplexityBot","Google-Extended","Google-CloudVertexBot","GPTBot","Diffbot","Meltwater","cohere-ai","Omgilibot","anthropic-ai","Bytespider","Amazonbot","YouBot","Meta-ExternalAgent","DuckAssistBot"]}
{"domain":"cnr.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"proofpoint.us","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zhqidown.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cms.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tilda.ws","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"gm.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"admaster.cc","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AdsBot-Google-Mobile","AdsBot-Google","PetalBot","dotbot","AhrefsBot"]}
{"domain":"dnsimple-edge.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"samokat.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"workdaysuv.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"cbsivideo.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hubspotusercontent-na1.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"inmarsat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vseinstrumenti.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"163jiasu.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","TimeoutError"]}
{"domain":"securityfocus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ebayimg.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zscloud.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"23.in-addr.arpa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"insales.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","Googlebot","*"]}
{"domain":"similarweb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wurl.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","AhrefsBot","SemrushBot","DotBot","BLEXBot","MJ12bot","PetalBot","MauiBot","DataForSeoBot"]}
{"domain":"jftechws.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"opsgenie.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"dev.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.cl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"howtogeek.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","\"008\"","anthropic-ai","Applebot","Applebot-extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","GPTBot","HTTrack","Nutch","Offline Explorer","omgili","Scrapy","YouBot"]}
{"domain":"playfamily.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"yp.to","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ttoverseaus.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bitwarden.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ClaudeBot","Claude-SearchBot","PerplexityBot","Amazonbot","anthropic-ai","Bytespider","CCBot"]}
{"domain":"bostonglobe.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":25,"stanza_tokens":["AmazonAdBot","AI2Bot-Dolma","Amazonbot","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot-Extended","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","Diffbot","FacebookBot","FirecrawlAgent","Google-Extended","GoogleOther","GPTBot","imageSpider","meta-externalagent","omgili","PanguBot","Spider","Timpibot","Webzio-Extended"]}
{"domain":"wordreference.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["Mediapartners-Google","Googlebot-Mobile","Googlebot","AdsBot-Google","AdsBot-Google-Mobile","Teoma","Slurp","Yeti","Yandex","NaverBot","bingbot","Applebot","Baiduspider","Sogou web spider","Opebot-v (https://www.1plusx.com)","AmazonAdBot","meta-externalagent","*"]}
{"domain":"ampproject.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID",200]}
{"domain":"razer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","CazoodleBot","MJ12bot","dotbot/1.0","Gigabot","Mediapartners-Google","AdsBot-Google","Googlebot-Image","Googlebot","Bingbot","Baiduspider","GPTBot","PerplexityBot","ClaudeBot","Google-Extended","YandexBot","Qwantify","SeznamBot","Yeti","Slurp"]}
{"domain":"bngf.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"jimdosite.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"ymmobi.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","ENOTFOUND"]}
{"domain":"wetteronline.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"chrome.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"awsdns-15.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gelbooru.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"straitstimes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot-Image","*"]}
{"domain":"azureedge.us","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"polyspeak.ai","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"selcdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"alphadnszone.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"custhelp.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"atl-paas.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"powerplatform.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"atom.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"razorpay.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","ia_archiver","ByteSpider","OAI-SearchBot","ChatGPT-User","PerplexityBot","FirecrawlAgent","AndiBot","ExaBot","PhindBot","YouBot","ClaudeBot","DuckAssistBot","Apple-Extended","Baiduspider","Yandex"]}
{"domain":"opensuse.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"dnsv2.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNREFUSED"]}
{"domain":"news24.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":48,"stanza_tokens":["*","Twitterbot","ia_archiver","MauiBot","chatgpt-user","bytespider","ccbot","diffbot","facebookbot","google-extended","gptbot","omgili","amazonbot","applebot","perplexitybot","youbot","008","dataprovider-com","dcrawl","httrack","httrack-3-0","metainspector","newspaper","nutch","offline-explorer","scrapy","ahrefsbot","barkrowler","blexbot","dataforseobot","domainstatsbot","dotbot","hypestat","linkdexbot","mj12bot","screaming-frog-seo-spider","semrushbot","semrushbot-ba","semrushbot-ct","semrushbot-si","semrushbot-swa","serpstatbot","zoombot","anthropic-ai","claude-web","claudebot","cohere-ai","Bytespider"]}
{"domain":"tass.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"apkpure.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot","Google-InspectionTool","Mediapartners-Google"]}
{"domain":"liberation.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":20,"stanza_tokens":["*","Googlebot-News","AI2Bot","Amazonbot","Applebot-Extended","anthropic-ai","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","Diffbot","DuckAssistBot","FacebookBot","GPTBot","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","PerplexityBot"]}
{"domain":"bdtelemetry.amazon","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-57.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"linksynergy.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"onenote.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"kraken.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zimbra.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","slurp","bingbot","Googlebot","Googlebot-image","MSNBot-media","MSNBot","Netcraft","Googlebot-Mobile","AdsBot-Google","Googlebot-Video","Googlebot-News","Mediapartners-Google","AdsBots-Google-Mobile","Rogerbot","PowerMapper"]}
{"domain":"uscourts.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sspnet.tech","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"ahagamecenter.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.com.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"adsbynimbus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gpubgm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ardmediathek.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":22,"stanza_tokens":["*","Amazonbot","applebot-extended","bytespider","ccbot","claudebot","google-extended","cohere-training-data-crawler","deepseekbot","deepseek","deepseek-crawler","diffbot","facebookbot","meta-externalagent","gptbot","grokbot","img2dataset","laiondownloader","mistralai-crawler","omgili","omgilibot","pangubot"]}
{"domain":"prestashop.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.dk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"dzeninfra.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"shopee.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot","Googlebot-Mobile","Bingbot","*"]}
{"domain":"toyota.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":22,"stanza_tokens":["GPTBot","ClaudeBot","anthropic-ai","Google-Extended","Applebot-Extended","Meta-ExternalAgent","meta-externalagent","Bytespider","CCBot","cohere-ai","Diffbot","Omgilibot","Amazonbot","DuckAssistBot","PerplexityBot","ChatGPT-User","Claude-User","Perplexity-User","OAI-SearchBot","Claude-SearchBot","Applebot","*"]}
{"domain":"lazada.com.ph","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"creditkarma.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mozilla/5.0 (compatible; Taboolabot/3.7; +http://www.taboola.com)","Twitterbot","*"]}
{"domain":"worldcat.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":5,"stanza_tokens":["GPTBot","ChatGPT-User","Google-Extended","CCBot","*"]}
{"domain":"gifshow.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["Baiduspider","Baiduspider-video","Googlebot","YoudaoBot","Sogou web spider","Sogou inst spider","360Spider","bingbot","*"]}
{"domain":"net.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"meb.gov.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":["OAI-SearchBot"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":40,"stanza_tokens":["ia_archiver","archive.org_bot","WaybackMachine","archive.ph","archive.today","special_archiver","Google-Extended","GPTBot","ChatGPT-User","CCBot","Bytespider","PerplexityBot","ClaudeBot","Applebot-Extended","Amazonbot","AhrefsBot","SemrushBot","MJ12bot","BLEXBot","DotBot","seoscanners.net","Screaming Frog SEO Spider","DataForSeoBot","ZoominfoBot","MauiBot","Barkrowler","SEOkicks-Robot","AspiegelBot","PetalBot","Exabot","DotBot/1.0","Omgilibot","FacebookBot","Applebot","anthropic-ai","Claude-Web","Diffbot","ImagesiftBot","Omgili","YouBot"]}
{"domain":"wordfence.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"osano.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","HubSpotContentSearchBot"]}
{"domain":"se.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"permutive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"menshealth.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AmazonAdBot","Google-InspectionTool","ChatGPT-User","GPTBot","OAI-SearchBot","*"]}
{"domain":"piano.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"serverdata.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"perl.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"1digitech.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"guardingvision.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"qvdt3feo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["* Disallow: /"]}
{"domain":"amazon.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"stocrm.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","*"]}
{"domain":"queniubg.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"clean.gg","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"codepen.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"awsdns-40.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zdxcloud.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"verizon.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR",200]}
{"domain":"awsdns-46.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"unhcr.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"awsdns-61.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"immobilienscout24.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Mediapartners-Google","OAI-SearchBot","ChatGPT-User","GPTBot","Claude-User","Claude-SearchBot","ClaudeBot","PerplexityBot"]}
{"domain":"google.com.pe","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"sportradarserving.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["GoogleOther","*"]}
{"domain":"keenetic.link","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"bpigagtm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-48.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pndsn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"connectad.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"skimresources.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"plausible.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"sproutsocial.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"dynamicyield.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"okko.tv","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"uiowa.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"register.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sat.gob.mx","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sfr.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":29,"stanza_tokens":["UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","larbin","libwww","ZyBORG","Download Ninja","wget","grub-client","k2spider","NPBot","WebReaper","Aboundexbot","Aboundex/0.2","*"]}
{"domain":"teacherspayteachers.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","meta-externalagent","CCBot","Yandex","ImagesiftBot","Slurp","Yahoo! Slurp","Twitterbot","Applebot-Extended"]}
{"domain":"awsdns-53.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"arm-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-01.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zoominfo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":57,"stanza_tokens":["*","Bingbot","Apple-Extended","ClaudeBot","FacebookBot","Google-Extended","GPTBot","Kangaroo Bot","PanguBot","ChatGPT-User","Operator","Applebot","OAI-SearchBot","PerplexityBot","Yandex","dotbot","rogerbot","moget","ichiro","NaverBot","Yeti","Baiduspider","Baiduspider-video","Baiduspider-image","sogou spider","YoudaoBot","MJ12bot","DJEcoBot","Amazonbot","PetalBot","AwarioSmartBot","AwarioRssBot","LinkedInBot","FreshpingBot","Pingdombot","coccocbot-web","coccocbot-image","sethbot","UptimeRobot","Mail.RU_Bot","heritrix","DuckAssistBot","YouBot","Meta-ExternalFetcher","Meta-ExternalAgent","Webzio-Extended","Timpibot","Omgili","Diffbot","cohere-training-data-crawler","Bytespider","CCBot","AI2Bot","NextGenSearchBot","ZoomBot","ZoominfoBot","DataForSeoBot"]}
{"domain":"routledge.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":35,"stanza_tokens":["*","AdsBot","Adsbot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","AdIdxBot","Applebot","APIs-Google","baiduspider","Bingbot","BingPreview","Caliperbot","CCBot","ContentKing","Deepcrawl","DuckDuckBot","facebot","facebookexternalhit/1.0","facebookexternalhit/1.1","Googlebot","Googlebot-Image","Googlebot-News","Googlebot-Video","LinkedInBot","MicrosoftPreview","Mediapartners-Google","Neevabot","PetalBot","SemrushBot","Slurp","Siteimprove","SemanticScholarBot","Storebot-Google","Twitterbot","YandexBot"]}
{"domain":"tdnsv8.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-09.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nextdns.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-23.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"icndns.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"livedoor.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"rte.ie","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":18,"stanza_tokens":["*","Mediapartners-Google","googlebot","FacebookBot","bingbot(at)microsoft.com","Bingbot","Baiduspider","YandexBot","Isposure Agent","proximic","AhrefsBot","Go-http-client","Giant Oak MN","magpie-crawler","GPTBot","ChatGPT-User","Google-Extended","CCBot"]}
{"domain":"eurekalert.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"fastlydns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-17.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"soufun.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"aws.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AdsBot-Google","AdsBot-Google-Mobile","*"]}
{"domain":"dnszi.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"aljazeera.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":10,"stanza_tokens":["*","AmazonAdBot","anthropic-ai","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","GPTBot","PerplexityBot","Bytespider"]}
{"domain":"travelandtourworld.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"fema.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"oregonstate.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"priceline.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["Mozilla/4.0 (compatible; MSIE 5.5; Windows NT; PCLN NOC SiteScope)","PATROL/V3.6.50i (Linux;INET KM 6.2.10 200410050940)","MJ12bot","AdIdxBot","008","searchpreview","adidxbot","adsbot-google","google-hoteladsverifier","Googlebot","OAI-SearchBot","OAI-AdsBot","*"]}
{"domain":"themegrill.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"amazon.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"google.fi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"servicepipe.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"amazon.sa","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"uscis.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","PetalBot"]}
{"domain":"brightspace.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":21,"stanza_tokens":["Amazonbot","anthropic-ai","Applebot-Extended","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","DuckAssistBot","FacebookBot","Google-CloudVertexBot","Google-Extended","GPTBot","Meta-ExternalAgent","meta-externalagent","PerplexityBot","Quora-Bot","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","*"]}
{"domain":"awsdns-05.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"jwpcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"twinrdengine.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"itau.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"grafana.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aoneroom.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"russianfood.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":21,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","dotbot","Yandex","sitebot","MJ12bot","wotbox","discoverybot","BLEXBot","AhrefsBot","XoviBot","SemrushBot","SemrushBot-SA"]}
{"domain":"familysearch.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Algolia Crawler","facebookexternalhit","Twitterbot","Mediapartners-Google*"]}
{"domain":"geforce.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aol.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":44,"stanza_tokens":["*","ADmantX","AlphaBot","anthropic-ai","AwarioRssBot","AwarioSmartBot","BLEXBot","Buzzbot","Bytespider","CCBot","ChatGPT-User","claritybot","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","huggingface","ImagesiftBot","img2dataset","magpie-crawler","Meltwater","Neevabot","news-please","NewsNow","Nutch","omgili","omgilibot","panscient.com","Perplexity-ai","PerplexityBot","PetalBot","PiplBot","scoop.it","Scrapy","Seekr","SentiBot","SeznamBot","TurnitinBot","YouBot","ZumBot"]}
{"domain":"picsart.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sii.cl","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"elevenlabs.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"lesechos.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":421,"stanza_tokens":["*","5emeRue","5erue","adequat","AI2Bot/v2.0","Aihitdata.com","Alexibot","AlphaSense","AlvinetSpider","amazon-kendra","Amazonbot","AmazonBot","AmiSoftware","Anomura","Antenne Hatena","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","ApocalXExplorerBot","Applebot","Applebot-Extended","Aranet-SearchBot","archive.org_bot","Argus","Ask n read","asknread.com","asterias","atlassian-bot","Augure","auramundi","AwarioRssBot","AwarioSmartBot","AzureAI-SearchBot","BackDoorBot/1.0","Barchart.com","BattleFin","bedrockbot/3.0","Bernin IT","bigsur.ai","BizInformation","Black Hole","Blackboard Safeassign","Bloodhound","BlowFish/1.0","BLP_bbot","bluematrix","BotALot","Brandwatch","Bravebot","Briefcase.news","Buck","BuiltBotTough","Bullseye/1.0","BunnySlippers","Bytespider","CCBo","CCBot","Cegbfeieh","Channel3Bot","ChatGLM-Spider","ChatGPT Agent","ChatGPT-User","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","CikisiBot","Cision","Claude-SearchBot","Claude-User","Claude-web","ClaudeBot","Cloudflare-AutoRAG","CloudVertexBot","coexel","cohere-ai","cohere-training-data-crawler","Comtex News Network","ConveraCrawler","CopyRightCheck","Corporama","cosmos","Cotoyogi","CotoyogiBot","Crawl4AI","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","cydralspider","D&B Hoovers","Data Expression","Data Observer","Dataminr","Datenbank Crawler","Dealogic","DeepSeekBot","Devin","Digimind","DirectFN","DISCo Pump 3.1","DittoSpyder"]}
{"domain":"gaijin.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"federalreserve.gov","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"leagueoflegends.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gcloudcs.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","TimeoutError"]}
{"domain":"talkie-ai.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"aruba.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"arbeitsagentur.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"evgnet.com","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":["ENOTFOUND",400]}
{"domain":"likee.video","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"samsungrs.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zjcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.no","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"google.co.nz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"stern.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":13,"stanza_tokens":["msnbot-media/1.1","msnbot-media/2.0b","Googlebot-News","scrapy","Diffbot","RyteBot","proximic","GPTBot","ChatGPT-User","Applebot-Extended","CCBot","Google-Extended","*"]}
{"domain":"volvocars.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":29,"stanza_tokens":["*","Google-Extended","CCBot","GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","Meta-ExternalAgent","FacebookBot","PerplexityBot","Perplexity-User","Applebot-Extended","Gemini-Deep-Research","Amazonbot","Bytespider","cohere-ai","YouBot","Omgilibot","Omgili","Diffbot","Googlebot","YisouSpider","Baiduspider","YandexBot","AhrefsBot","PetalBot"]}
{"domain":"nos.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":26,"stanza_tokens":["*","CCBot","ChatGPT-User","GPTBot","Google-Extended","Google-CloudVertexBot","Applebot-Extended","anthropic-ai","ClaudeBot","Omgilibot","Omgili","FacebookBot","Diffbot","DuckAssistBot","AI2Bot","Bytespider","Kangaroo Bot","ImagesiftBot","PerplexityBot","cohere-ai","Meta-ExternalAgent","Meta-ExternalFetcher","Timpibot","Webzio-Extended","YouBot","DeepSeek"]}
{"domain":"lazada.sg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zenodo.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mydnspt.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"mp.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","TimeoutError"]}
{"domain":"fivecdm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-56.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"clarium.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bell-labs.com","fetch_ok":false,"usable":false,"reason":"other","status":410,"error":null,"attempts":[410,410]}
{"domain":"apnic.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gamma.app","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tinypass.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"logicnow.us","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wufoo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"chinanetcenter.com","fetch_ok":false,"usable":false,"reason":"other","status":406,"error":null,"attempts":["ENOTFOUND",406]}
{"domain":"unfccc.int","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"2o7.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"paypalobjects.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"srv53.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"kemenag.go.id","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","TimeoutError"]}
{"domain":"ktalk.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"3.in-addr.arpa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"3hentai.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":8,"stanza_tokens":["*","SEOkicks-Robot","ia_archiver","archive.org_bot","special_archiver","SemrushBot","SemrushBot-SA","GPTBot"]}
{"domain":"screenrant.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cableone.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":[429,429,"TimeoutError"]}
{"domain":"zaloapp.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"second-ns.de","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tuyaus.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bitrix.info","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"liteapks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vkplay.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot","Yandex"]}
{"domain":"sprig.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ctlcdn.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mercari.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["AdsBot-Google-Mobile","Googlebot-Image","AdsBot-Google","Googlebot","PetalBot","YandexBot","*"]}
{"domain":"cars.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"cisa.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","PetalBot"]}
{"domain":"perimeterx.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ECONNRESET"]}
{"domain":"awsdns-58.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ettdnsv.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cian.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"viafoura.co","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ECONNRESET"]}
{"domain":"seattletimes.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"amazon.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"kp.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","Googlebot"]}
{"domain":"cxense.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"poshmark.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Pinterestbot"]}
{"domain":"filmweb.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"mintegral.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mgts.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"awsdns-21.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"f-sos.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vecteezy.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"gname-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"onenote.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"owasp.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"centurygame.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-44.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"weborama.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"anu.edu.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"visualcapitalist.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","bingbot","LinkedInBot"]}
{"domain":"nbc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dreame.tech","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"matchtv.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"acobt.tech","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"thestar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":94,"stanza_tokens":["AI2Bot","Ai2Bot-Dolma","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","Diffbot","FacebookBot","FriendlyCrawler","GPTBot","Google-Extended","ICC-Crawler","ImagesiftBot","Meta-ExternalAgent","OAI-SearchBot","PerplexityBot","PetalBot","Scrapy","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot","anthropic-ai","cohere-ai","iaskspider/2.0","img2dataset","omgili","omgilibot","cohere-training-data-crawler","Kangaroo Bot","PanguBot","Amazonbot","DuckAssistBot","Meta-ExternalFetcher","Perplexity-User","Claude-SearchBot","Claude-User","Cotoyogi","Datenbank","Devin","Factset_spyderbot","Gemini-Deep-Research","MistralAI-User","netEstate","Imprint","SemrushBot-OCOB","AddSearchBot","bigsur.ai","CloudVertexBot","GoogleOther","LinerBot","meta-externalagent","QualifiedBot","Ai2Bot-DeepResearchEval","ChatGLM-Spider","DeepSeekBot","Google-NotebookLM","iAskBot","KunatoCrawler","meta-externalfetcher","meta-webindexer","webzio-extended","WRTNBot","ZanistaBot","AI2Bot-DeepResearchEval","Anomura","atlassian-bot","Bravebot","Channel3Bot","Cloudflare-AutoRAG","Crawl4AI","Google-CloudVertexBot","imageSpider","KlaviyoAIBot","laion-huggingface-processor","LCC","LinkupBot","PhindBot","Poggio-Citations","SBIntuitionsBot","TavilyBot","Amzn-SearchBot","Amzn-User","ApifyBot","ApifyWebsiteContentCrawler","Aranet-SearchBot","AzureAI-SearchBot","ExaBot","FirecrawlAgent","kagi-fetcher","TerraCotta","*"]}
{"domain":"coindesk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":23,"stanza_tokens":["*","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","Meta-ExternalAgent","meta-externalagent","omgili","omgilibot","PerplexityBot","Quora-Bot","Scrapy","TurnitinBot"]}
{"domain":"awsdns-31.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"proxad.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"granbluefantasy.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"capitaloneshopping.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"penguinrandomhouse.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wargaming.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"www.gov.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-06.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"emxdgt.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"dbankcloud.asia","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"ucloud.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"qwest.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"statsigapi.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"awsdns-45.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"volcgtm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nbcuni.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","* Disallow:"]}
{"domain":"kaufland.de","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ispconfig.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sfr.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"replit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","FreshpingBot"]}
{"domain":"intergient.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNRESET"]}
{"domain":"gesetze-im-internet.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"11st.co.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","Mediapartners-Google","Googlebot","Googlebot-image","NaverBot","Yeti","Daumoa","Twitterbot","OAI-SearchBot","ChatGPT-User","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","Google-Extended","ads-naver"]}
{"domain":"taxssp.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"browsiprod.com","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":[400,"ENOTFOUND"]}
{"domain":"easyjet.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pantheonsite.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["*","RavenCrawler","rogerbot","dotbot","SemrushBot","SiteAuditBot","SplitSignalBot","PowerMapper","Swiftbot","lyticsbot","Dubbotbot","PopeTech-CrawlBot","PopeTech-ScanBot"]}
{"domain":"codeberg.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":29,"stanza_tokens":["*","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","Diffbot","FacebookBot","facebookexternalhit","FriendlyCrawler","Google-Extended","GPTBot","ICC-Crawler","ImagesiftBot","img2dataset","meta-externalagent","OAI-SearchBot","Omgili","Omgilibot","PerplexityBot","PetalBot","Scrapy","Timpibot","VelenPublicWebCrawler","YouBot"]}
{"domain":"netbird.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hostingersite.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot","*"]}
{"domain":"technoratimedia.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"awsdns-14.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"fandango.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"mktoresp.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"google.com.pk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"comodo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","rogerbot"]}
{"domain":"getadblock.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mcr-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sfdcfc.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"umass.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","SemrushBot"]}
{"domain":"cyberark.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"126.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"dinahosting.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","PerplexityBot","Applebot-Extended","Bytespider","CCBot","cohere-ai","Meta-ExternalAgent","OAI-SearchBot"]}
{"domain":"ndash.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vaticannews.va","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"ksyna.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"spamcop.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.ae","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"firebaseapp.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"mailerlite.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"findagrave.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"zimperium.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"realestate.com.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":30,"stanza_tokens":["*","googlebot-news","YakazBot","trovitBot","Eyeotabot","PiplBot","CriteoBot/0.1","trendkite-akashic-crawler","GPTBot","CCBot","anthropic-ai","cohere-ai","ia_archiver","omgili","omgilibot","MJ12bot","meta-externalagent","ClaudeBot","Claude-User","Claude-SearchBot","Bytespider","PerplexityBot","Perplexity-User","Ai2Bot","Kangaroo Bot","PanguBot","Amazonbot","Applebot","Applebot-Extended","bingbot"]}
{"domain":"paysafecard.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-04.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"growcdnssedge.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"uoregon.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rfi.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":326,"stanza_tokens":["*","Googlebot","AI2Bot","Ai2Bot-Dolma","AlibabaBot","AliyunBot","anthropic-ai","Baidu-YunGuanCe","Big Sur AI","ByteDance-AI","ChatGPT Operator","ChatGPT-User","Claude-Code","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","DeepSeekBot","DigitalOceanGenAICrawler","DuckAssistBot","ERNIEBot","ERNIE-Bot","Google-Extended","GPTBot","Grok","img2dataset","Kimibot","KimiCrawler","LINER Bot","Meta Llama","MetaAIBot","Moonshot-AI","MoonshotBot","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","OAI-SearchBot","PanguBot","Perplexity-User","Perplexity-User/1.0","PerplexityBot","quillbot.com","QwenBot","SBIntuitionsBot","TongyiBot","Tongyi-Crawler","YiyanBot","YouBot","008","5emeRue","5erue","adequat","adequat-systems","ADmantX bot","AhrefsBot","Alexibot","AlvinetSpider","AmiSoftware","Antenne Hatena","ApocalXExplorerBot","Argus","Ask n read","asknread.com","asterias","Augure","augure","auramundi","Babya Discoverer","BackDoorBot/1.0","BacklinkCrawler","bitlybot","BizInformation","Black Hole","Blekkobot","BLEXBot","Bloodhound","BlowFish/1.0","BotALot","BuiltBotTough","Bullseye/1.0","BunnySlippers","CCBot","Cegbfeieh","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","Cision","Cliqzbot","coexel","ConveraCrawler","CopyRightCheck","Corporama","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","cydralspider","Digimind"]}
{"domain":"grandviewresearch.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"libbyshelf.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"infolinks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gulfnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot","Googlebot-Mobile","AmazonAdBot"]}
{"domain":"ilsole24ore.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["anthropic-ai"],"open":["ClaudeBot","Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":18,"stanza_tokens":["*","GPTBot","CCBot","anthropic-ai","Google-Extended","PetalBot","Yandex","Amazonbot","Applebot","OAI-SearchBot","PerplexityBot","QualifiedBot","ICC Crawler","Meta-ExternalAgent","Factset_spyderbot","ChatGPT-User","Perplexity-User","DuckAssistbot"]}
{"domain":"zuimeitianqi.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"uuidksinc.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"humeysha.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-18.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hiyaapi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ndmdhs.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"banahosting.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Googlebot","AdsBot-Google","Googlebot-Image"]}
{"domain":"tunein.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"findlaw.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"espncdn.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",202]}
{"domain":"airasia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"vidazoo.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"kingsgroupgames.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,530]}
{"domain":"awsdns-02.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"openoffice.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["daumoa","Sosospider","Snoopy","Browsershots","*"]}
{"domain":"callrail.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"cquotient.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ECONNRESET"]}
{"domain":"alice.it","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_VERIFY_LEAF_SIGNATURE","attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE","UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"news18.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pippio.com","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":[503,"ENOTFOUND",503]}
{"domain":"simplisafe.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"seek.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"complianz.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"handle.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dns.pt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"quad9.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"queniuyk.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"reolink.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Googlebot","Googlebot-Image","AdsBot-Google","*","PowerMapper","Amazonbot"]}
{"domain":"serpro.gov.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"awsdns-63.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hookspf.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"6sc.co","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"knowbe4.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"emojipedia.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Google-Extended","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","GPTBot","meta-externalagent"]}
{"domain":"carfax.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aif.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","Bingbot"]}
{"domain":"google.com.sa","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"steaminventoryhelper.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-55.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"alltel24.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"proxytg.space","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vigo.tech","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-19.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-28.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tfl.gov.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"rei.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"lifeaiot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot","*"]}
{"domain":"swrve.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",200]}
{"domain":"glulive.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"canvas-user-content.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"medallia.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"line-scdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"livehindustan.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","Googlebot-News","facebookexternalhit","Facebot","Google-Extended","Mediapartners-Google","AdsBot-Google","AmazonAdBot"]}
{"domain":"chewy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Amazonbot","AdsBot-Google","adidxbot"]}
{"domain":"helpshift.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"udn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":7,"stanza_tokens":["Meta-ExternalAgent","Meta-ExternalFetcher","facebookexternalhit","*","GPTBot","Claude","ClaudeBot"]}
{"domain":"elespanol.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":111,"stanza_tokens":["*","grapeshot","AmazonAdBot","008 Disallow: /","AddThis.com Disallow: /","admantx Disallow: /","AhrefsBot Disallow: /","BDCbot Disallow: /","Bender Disallow: /","BIXOCRAWLER Disallow: /","bl.uk_lddc_bot Disallow: /","BLEXBot Disallow: /","BUbiNG Disallow: /","Cliqzbot Disallow: /","CNCDialer Disallow: /","crawler4j Disallow: /","CrystalSemanticsBot Disallow: /","CyberAlert Disallow: /","DigExt Disallow: /","discobot Disallow: /","discoverybot Disallow: /","dloader Disallow: /","dloader(NaverRobot) Disallow: /","DOC Disallow: /","dotbot Disallow: /","Download Ninja Disallow: /","DTS Agent Disallow: /","Exabot Disallow: /","Ezooms Disallow: /","FairShare Disallow: /","Fetch Disallow: /","Flamingo_SearchEngine Disallow: /","Genieo Disallow: /","Gigabot Disallow: /","grub-client Disallow: /","Heritrix Disallow: /","heritrix/3.3.0 Disallow: /","HTTrack Disallow: /","ia_archiver Disallow: /","integromedb Disallow: /","IstellaBot Disallow: /","JikeSpider Disallow: /","Jyxobot Disallow: /","k2spider Disallow: /","Kimengi Disallow: /","Kimengi/nineconnections.com Disallow: /","larbin Disallow: /","LexxeBot/1.0 Disallow: /","libwww Disallow: /","linko Disallow: /","Livelapbot Disallow: /","magpie-crawler Disallow: /","Maxthon Disallow: /","MetaURI Disallow: /","Microsoft.URL.Control Disallow: /","MJ12bot Disallow: /","Moreover Disallow: /","Moreoverbot Disallow: /","MSIECrawler Disallow: /","nabot Disallow: /","NaverBot Disallow: /","NerdByNature.Bot Disallow: /","netEstate NE Crawler Disallow: /","NetSeer crawler Disallow: /","Newscan Disallow: /","NextGenSearchBot Disallow: /","NPBot Disallow: /","nutch Disallow: /","Offline Explorer Disallow: /","omgilibot Disallow: /","Orthogaffe Disallow: /","PiplBot Disallow: /","Pixray-Seeker Disallow: /","proximic Disallow: /","psbot Disallow: /","QuerySeekerSpider Disallow: /","rogerbot Disallow: /","seokicks Disallow: /","SEOkicks-Robot Disallow: /","SiteBot Disallow: /","SiteBot/0.1 Disallow: /","sitecheck.internetseer.com Disallow: /","SiteSnagger Disallow: /","Slurp Disallow: /","sogou Disallow: /","Sosospider Disallow: /","spbot Disallow: /","Spinn3r Disallow: /","Teleport Disallow: /","TeleportPro Disallow: /","trendictionbot Disallow: /","trovitBot Disallow: /","TurnitinBot Disallow: /","UbiCrawler Disallow: /","uMBot-LN Disallow: /","UnisterBot Disallow: /","UniversalFeedParser Disallow: /","WBSearchBot Disallow: /","WebCopier Disallow: /","WebReaper Disallow: /"]}
{"domain":"sephora.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sardine.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"n8n.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xhpingcdn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["Googlebot","Mediapartners-Google","Yahoo! Slurp","bingbot","Yandex","Baiduspider","Sogou","ia_archiver","IPS-Agent","BLEXBot","Pandalytics","IonCrawl","*"]}
{"domain":"awsdns-60.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"clickpetroleoegas.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bcebos.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-50.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"goethe.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unsw.edu.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["SiteimproveBot","SiteimproveBot-Crawler","*"]}
{"domain":"pchome.com.tw","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"missena.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"uc.cn","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"steptofsvo.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"todoist.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"js.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"videotron.ca","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"tns-counter.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"tmtreader.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"prismaaccess.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"plala.or.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"mainichi.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":30,"stanza_tokens":["CCBot","GPTBot","ChatGPT-User","OAI-SearchBot","Google-Extended","ICC-Crawler","ClaudeBot","anthropic-ai","Claude-Web","PerplexityBot","Perplexity-ai","Applebot-Extended","cohere-ai","Meta-ExternalAgent","Gensparkbot","Bytespider","omgili","omgilibot","cohere-training-data-crawler","Diffbot","FacebookBot","magpie-crawler","meta-externalfetcher","Scrapy","Timpibot","Webzio-Extended","AmazonBot","Claude-SearchBot","Claude-User","AI2Bot"]}
{"domain":"gwu.edu","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"twtrdns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"lunalabs.io","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":[503,503,503]}
{"domain":"microsoftapp.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"channel4.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"ghost.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"backblaze.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"verywellhealth.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"kia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["ia_archiver","*"]}
{"domain":"proxysite.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"op.gg","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot","Bingbot","carbon-umbrella-bot","*"]}
{"domain":"awin1.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"brandmetrics.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"clmbtech.com","fetch_ok":false,"usable":false,"reason":"other","status":502,"error":null,"attempts":[502,"ENOTFOUND",502]}
{"domain":"netsuite.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"fsu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"virginia.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mysterium.network","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AdsBot-Google-Mobile","AdsBot-Google","PetalBot","dotbot","AhrefsBot"]}
{"domain":"kompas.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":86,"stanza_tokens":["*","Twitterbot","Twitterbot/1.0","Xbot","X-Twitter-Image-Crawler","Amazonbot","GPTBot","OpenAI","ChatGPT-User","CCBot","anthropic-ai","Claude-Web","PerplexityBot","Applebot-Extended","AudigentAdBot","AwarioRssBot","AwarioSmartBot","BLEXBot","Bytespider","ClaudeBot","cohere-ai","DataForSeoBot","Diffbot","DuckAssistBot","EchoboxBot","FacebookBot","FriendlyCrawler","ImagesiftBot","Jetslide","magpie-crawler","Meta-ExternalAgent","meta-externalagent","Meta-ExternalFetcher","meta-externalfetcher","Meta-WebIndexer","MistralAI-User","MyCentralAIScraperBot","NewsNow","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","Poseidon Research Crawler","quillbot.com","Quora-Bot","Scrapy","SeekrBot","SeznamHomepageCrawler","TaraGroup Intelligent Bot","Timpibot","TurnitinBot","ViennaTinyBot","YouBot","newspaper","Perplexity-User","docomo","yisouspider","trendictionbot","tin_eye","metaexternalagent","chatgpt_user","Claude-User","Claude-SearchBot","ias-va","ias-or","ias-ir","ias-jp","ias-sg","ias-au","ias_crawler","admantx","Webz.io","ICC-Crawler","you.com.","ChatGPT’s agent Operator.","Bing’s Copilot chat.","Grok.","archive.org_bot","archive_bot","ia_archiver","Baiduspider","baiduspider","Baiduspider-image","Baiduspider-video"]}
{"domain":"appboy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cvs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["*","AdsBot-Google","Googlebot-image","Yandex","Prerender","CCBot","ChatGPT-User","GPTBot","anthropic-ai","Google-Extended","ClaudeBot","Claude-Web","cohere-ai"]}
{"domain":"pajak.go.id","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"japanpost.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dickssportinggoods.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"onamae.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","OAI-AdsBot","OAI-SearchBot","GPTBot","PerplexityBot"]}
{"domain":"propublica.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"computerworld.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":37,"stanza_tokens":["*","Ai2Bot-Dolma","AI2Bot","anthropic-ai","Applebot-Extended","Arquivo-web-crawler","AwarioRssBot","AwarioSmartBot","Barkrowler","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","crawler4j","dotbot","Diffbot","FacebookBot","FriendlyCrawler","GPTBot","https://hada.news","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","MJ12bot","PanguBot","Scrapy","Seekr","Timpibot","VelenPublicWebCrawler","Webzio-Extended"]}
{"domain":"mux.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mogujie.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"g2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":46,"stanza_tokens":["*","GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","Meta-ExternalAgent","Amazonbot","CCBot","bingbot","bingpreview","AhrefsBot","SemrushBot","Diffbot","UbiCrawler","BUbiNG","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","wget","grub-client","k2spider","NPBot","WebReaper","rogerbot","SemrushBot-SA","BrandVerity","Bytespider","DeepSeekBot"]}
{"domain":"apachelounge.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"a-ads.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"consentmanager.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"pipopay.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"fotostrana.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"answers.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","Applebot","Googlebot","bingbot","msnbot","Slurp","Facebot","baiduspider","sosobot","exabot","Mediapartners-Google"]}
{"domain":"southwest.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AhrefsBot","Screaming Frog SEO Spider"]}
{"domain":"adyen.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"verywellmind.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"appnext.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"cbr.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-38.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wikipedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["MJ12bot","Mediapartners-Google*","IsraBot","Orthogaffe","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","fast","wget","grub-client","k2spider","NPBot","WebReaper","SemrushBot","*"]}
{"domain":"69shuba.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["Baiduspider","Sosospider","sogou spider","SemrushBot","AhrefsBot","PetalBot","*"]}
{"domain":"snopes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":["OAI-SearchBot"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":32,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","GPTBot","meta-externalagent","anthropic-ai","Applebot","AwarioRssBot","AwarioSmartBot","ChatGPT-User","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","FacebookBot","FriendlyCrawler","img2dataset","ImagesiftBot","magpie-crawler","Meltwater","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","PiplBot","scoop.it","Seekr","YouBot"]}
{"domain":"clara.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","GPTBot","Google-Extended","CCbot","ClaudeBot"]}
{"domain":"sina.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"noon.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"farfetch.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AdsBot-Google","*","Yandex"]}
{"domain":"proxmox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adtarget.biz","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"judge.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"udel.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"abril.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","googlebot","proximic"]}
{"domain":"hbo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bayern.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mcusercontent.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Twitterbot","facebookexternalhit","*"]}
{"domain":"livemint.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","facebookexternalhit","Facebot","Google-Extended","AmazonAdBot","AdsBot-Google","Mediapartners-Google"]}
{"domain":"52shuku.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"merchant-center-analytics.goog","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"pananames.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"searchengineland.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"redcross.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"raspberrypi.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cgtn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"marinetraffic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"uefa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vkusvill.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rtmark.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ENOTFOUND"]}
{"domain":"your-storageshare.de","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"queniusz.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"p-cdn.us","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"xn--ngstr-lra8j.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"netshoes.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"zerocdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ECONNREFUSED","ECONNREFUSED"]}
{"domain":"syndicatedsearch.goog","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND",404]}
{"domain":"next.co.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"livechat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","mauibot"]}
{"domain":"alamy.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":["TimeoutError",429,429]}
{"domain":"hse.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":4,"stanza_tokens":["*","GPTBot","Googlebot","Yandex"]}
{"domain":"googlesource.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"vbnmhjlp.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"tvtropes.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":["OAI-SearchBot"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":33,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","GPTBot","meta-externalagent","anthropic-ai","Applebot","Awario*","ChatGPT-User","Claude*","cohere-ai","DataForSeoBot","Diffbot","FacebookBot","FriendlyCrawler","img2dataset","ImagesiftBot","magpie-crawler","Meltwater","NeevaBot","omgili*","peer39_crawler","PerplexityBot","PiplBot","scoop.it","Seekr","YouBot","AhrefsBot","SemrushBot","MJ12bot"]}
{"domain":"abtasty.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"japantimes.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ctrip.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"heytapimg.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"boe.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"up.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"google.co.ke","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"my.games","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"voskhod.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fastecn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bldrdoc.gov","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"openweathermap.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","Googlebot"]}
{"domain":"zlib.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dmca.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rwth-aachen.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ctripgslb.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nbcuott.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"edgedns-tm.info","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fnac.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"gap.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"nontondrama.my","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"instana.io","fetch_ok":false,"usable":false,"reason":"other","status":401,"error":null,"attempts":[401,"ENOTFOUND"]}
{"domain":"novelbin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"popularmechanics.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AmazonAdBot","Google-InspectionTool","ChatGPT-User","GPTBot","OAI-SearchBot","*"]}
{"domain":"indriverapp.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mk.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","*"]}
{"domain":"boardgamegeek.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":48,"stanza_tokens":["*","MJ12Bot","PetalBot","AspiegelBot","AhrefsBot","SEMrushBot","DotBot","MauiBot","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot-Extended","Brightbot 1.0","Bytespider","CCBot","ChatGLM","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Crawlspace","DeepSeekBot","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","imgproxy","ISSCyberRiskCrawler","Kangaroo Bot","Meta-ExternalAgent","omgili","omgilibot","PanguBot","Perplexity-User","PerplexityBot","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot"]}
{"domain":"ahajournals.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":5,"stanza_tokens":["*","facebookexternalhit","LinkedInBot","Twitterbot","GPTBot"]}
{"domain":"hyundai.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["Googlebot","Bingbot","OAI-SearchBot","ChatGPT-User","Google-Extended","GPTBot","ClaudeBot","Claude-SearchBot","Claude-User","PerplexityBot","GoogleOther","BingPreview","*","ScrapeHero"]}
{"domain":"google.co.ve","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"vanderbilt.edu","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"cursor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thenextweb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","googlebot-news"]}
{"domain":"google.pt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"mercedes-benz.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"faa.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"squid-cache.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_GET_ISSUER_CERT_LOCALLY","attempts":["UNABLE_TO_GET_ISSUER_CERT_LOCALLY","UNABLE_TO_GET_ISSUER_CERT_LOCALLY"]}
{"domain":"macrumors.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"growthbook.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"arlo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":24,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","anthropic-ai","Claude-Web","PerplexityBot","Perplexity-User","Google-Extended","Applebot-Extended","Amazonbot","cohere-ai","CCBot","Diffbot","Bytespider","meta-externalagent","FacebookBot","adsbot-google","Nutch","AhrefsBot","AhrefsSiteAudit","MJ12bot","Pinterest"]}
{"domain":"football.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"juntadeandalucia.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","PetalBot"]}
{"domain":"producthunt.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.com.bd","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"cwi.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"arin.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"elster.de","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"qlogo.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vietjack.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"britishcouncil.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"linkis.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"outbrainimg.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"torproject.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tracker.cl","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ntu.edu.tw","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"venmo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-00.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"yaeby.work","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"mosspf.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"defense.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"UND_ERR_CONNECT_TIMEOUT"]}
{"domain":"tiktok-row.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"nssvc.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sportskeeda.com","fetch_ok":false,"usable":false,"reason":"other","status":405,"error":null,"attempts":[405,405]}
{"domain":"ezoic.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"byte008.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"iu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AdsBot-Google-Mobile","AdsBot-Google","*"]}
{"domain":"ucsf.edu","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"clickagy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dji.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"people.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sexvid.pro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["BingBot","*"]}
{"domain":"joinhoney.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AdsBot-Google"]}
{"domain":"spaceship.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["dotbot"]}
{"domain":"eia.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"termly.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Scrapy","*"]}
{"domain":"flipsnack.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","CCBot","Bytespider"]}
{"domain":"chromium.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bakecaincontrii.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mediarithmics.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"myhostadmin.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*��"]}
{"domain":"opentable.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"uxfeedback.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UND_ERR_CONNECT_TIMEOUT","attempts":["UND_ERR_CONNECT_TIMEOUT","ENOTFOUND"]}
{"domain":"buymeacoffee.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"buffer.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"photofolio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","Claude-User","Claude-SearchBot","Claude-Web","PerplexityBot","Perplexity-User","DuckAssistBot","Amazonbot","MJ12bot"]}
{"domain":"veeam.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"spreaker.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dmcdn.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"abb.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"privacysandboxservices.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"af.mil","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"unioneeu.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"protek.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nrk.no","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":56,"stanza_tokens":["*","AppleNewsBot","Googlebot-News","Amazonbot","anthropic-ai","Bytespider","ChatGPT-User","Claude-Web","cohere-ai","Diffbot","FacebookBot","Google-Extended","ImagesiftBot","NewsNow","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","PerplexityUser","Scrapy","AI2Bot","amazon-kendra","Applebot-Extended","Bedrock-bot","CCBot","ChatGLM-Spider","Claude-SearchBot","Claude-User","ClaudeBot","Cotoyogi","DeepSeek","DuckAssistBot","EchoboxBot","Gemini-Deep-Research","Google-CloudVertexBot","GPTBot","iAskBot","ICC-Crawler","Kunato","LinerBot","Meta-ExternalAgent","Meta-ExternalFetcher","mistral.ai","MistralAI-User","PanguBot","ProRataInc","QualifiedBot","SeekrBot","TikTokSpider","VelenPublicWebCrawler","WARDBot","WPBot","WrtnBot"]}
{"domain":"visx.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"tum.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"leadconnectorhq.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"forticloud.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"sonicwall.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ssc.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"kunlunaq.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"kinghost.com.br","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"avto.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"azioncdn.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"gdz.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"lovable.app","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"shopee.co.th","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"maxesads.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"studentaid.gov","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"wpscdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"midasplayer.cloud","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"ahcdn.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"steamconnecttest.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"adfox.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"sape.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","UND_ERR_CONNECT_TIMEOUT"]}
{"domain":"topdns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"united.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"miibeian.gov.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"tcgplayer.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"awsdns-32.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"vzwwo.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"movetv.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"51.la","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"boomplaymusic.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"milanuncios.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"theiconic.com.au","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"purl.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"siteforce.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["googlebot","*"]}
{"domain":"ucweb.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","UND_ERR_CONNECT_TIMEOUT","TimeoutError"]}
{"domain":"blender.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gdz-raketa.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","GoogleBot","Yandex"]}
{"domain":"walmart.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Bingbot"]}
{"domain":"finam.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"iotbing.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"google.com.do","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"uw.edu","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cloudflaressl.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"xarxx.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"derstandard.at","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":15,"stanza_tokens":["Applebot-Extended","anthropic-ai","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","FacebookBot","Google-Extended","GPTBot","ia_archiver","OAI-SearchBot","omgili","omgilibot","PerplexityBot","PerplexityUser"]}
{"domain":"speedtestcustom.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"nic.fr","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_VERIFY_LEAF_SIGNATURE","attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE","UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"aliexpress.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","AdsBot-Google","Googlebot","Yandex"]}
{"domain":"ivideon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"shutterfly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","GoogleBot","R6_FeedFetcher","Genieo","genieo","Pinterestbot","deepcrawl"]}
{"domain":"nordstrom.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["AdsBot-Google","Googlebot-Image","Googlebot","*"]}
{"domain":"owhealth.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",404]}
{"domain":"xn--69-6tia3cb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"tlivesdk.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"phpbb.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"jst.go.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"shopee.tw","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot","Googlebot-Mobile","Bingbot","*"]}
{"domain":"alarabiya.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"xxxpostpic.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vk-portal.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"SELF_SIGNED_CERT_IN_CHAIN","attempts":["SELF_SIGNED_CERT_IN_CHAIN","ENOTFOUND"]}
{"domain":"videyli.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"bidderstack.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wp.me","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"20minutos.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":120,"stanza_tokens":["*","proximic","Yandex","Baiduspider","008","AddThis.com","admantx","BDCbot","Bender","bl.uk_lddc_bot","BLEXBot","BUbiNG","CNCDialer","CrystalSemanticsBot","CyberAlert","discobot","discoverybot","DOC","DotBot","Download Ninja","Exabot","Ezooms","Fetch","Flamingo_SearchEngine","grub-client","heritrix","HTTrack","integromedb","IstellaBot","JikeSpider","Jyxobot","k2spider","larbin","LexxeBot/1.0","libwww","linko","magpie-crawler","Maxthon","MJ12bot","Moreoverbot","MSIECrawler","NerdByNature.Bot","netEstate NE Crawler","NetSeer crawler","Newscan","NextGenSearchBot","NPBot","Offline Explorer","Pixray-Seeker","psbot","rogerbot","seokicks","SiteBot","SiteBot/0.1","sitecheck.internetseer.com","SiteSnagger","sogou","Sosospider","spbot","Spinn3r","Teleport","TeleportPro","trovitBot","TurnitinBot","UbiCrawler","UnisterBot","WBSearchBot","WebCopier","WebReaper","WebStripper","WebZIP","WeSEE:Search","wget","wotbox","Yasni","Zao","Zealbot","ZyBORG","anthropic-ai","AI2Bot","Ai2Bot-Dolma","AmazonBot","ApifyBot","ApifyWebsiteContentCrawler","Applebot-Extended","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Diffbot","FacebookBot","FirecrawlAgent","GPTBot","iAskBot"]}
{"domain":"google.com.ng","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"eclipse.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"expressapisv2.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"techlab-cdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"thekamababa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"xn--3-nyf3aak0c.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"kolkataff.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sanity.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"easydns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",404]}
{"domain":"hdhub4u.fo","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"playinjeet.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"spboss.mobi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot","*"]}
{"domain":"eu.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"page.link","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":[400,400]}
{"domain":"bitdrift.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"u-tokyo.ac.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNRESET"]}
{"domain":"cbrpay.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"virtualmin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"db.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"popsci.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"betpawa.cm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","ClaudeBot","anthropic-ai","PerplexityBot","Amazonbot","Bytespider","cohere-ai"]}
{"domain":"betway.co.bw","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"formula1.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"alltrails.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":69,"stanza_tokens":["Applebot-Extended","Atomic_Email_Hunter","Bender","Birubot","BlogPulseLive","BOTW Spider","Butterfly","CCBot","ChatGPT-User","Claude-SearchBot","Claude-User","ClaudeBot","Crowsnest","discobot","DoCoMo","dotbot","Exabot","expo9","Ezooms","FairShare","Feedtrace","Ginxbot","Google-Extended","gosospider","GPTBot","knowaboutBot","Meta-ExternalAgent","MetaURI","MJ12bot","MLBot","Moreoverbot","mxbot","my6sense","netseer","NHN Corp","NjuiceBot","Nutch","OAI-AdsBot","OAI-SearchBot","OneRiot","percbotspider","PerplexityBot","Perplexity-User","Postrank","proximic","Purebot","radian6","ScoutJet","semiocast","Showyoubot","SiteBot","suggybot","Summify","Tagoobot","TANNER","ThingFetcher","trendictionbot","TweetedTimes Bot","TweetmemeBot","Twiceler","Twingly Recon","twitmatic","Twitturls","Twitturly","Typhoeus","Viralheat","YodaoBot","YouBot","*"]}
{"domain":"betcity.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sky.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dbankcdn.cn","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"scw.cloud","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_PACKET_LENGTH_TOO_LONG","attempts":["ERR_SSL_PACKET_LENGTH_TOO_LONG","ERR_SSL_PACKET_LENGTH_TOO_LONG"]}
{"domain":"ucoz.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"virtua.com.br","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"moviesda19.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"whost-solutions.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dnsswitch.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","UND_ERR_CONNECT_TIMEOUT","ENOTFOUND"]}
{"domain":"backblazeb2.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"cdnhwc6.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"firstbytedns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"keepverify.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sciencealert.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"railway.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AdsBot-Google","Googlebot"]}
{"domain":"ebaycdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gb.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"fitbit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"medal.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"foxsports.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"chanel.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ugtelset.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_PACKET_LENGTH_TOO_LONG","attempts":["ERR_SSL_PACKET_LENGTH_TOO_LONG","ERR_SSL_PACKET_LENGTH_TOO_LONG"]}
{"domain":"bsclink.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"bcg.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"r7ops.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"adnxs-simple.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"echostarcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"neilpatel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","SiteCheckerBotCrawler","SiteChecker"]}
{"domain":"mobilityware.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"anwap.today","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","CERT_HAS_EXPIRED"]}
{"domain":"tk0x1.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND",204]}
{"domain":"syncthing.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"operacdn.com","fetch_ok":false,"usable":false,"reason":"other","status":204,"error":null,"attempts":[204]}
{"domain":"carrd.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cdn-cookieyes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":13,"stanza_tokens":["Googlebot","Bingbot","Applebot","AhrefsBot","Amazonbot","Bytespider","PetalBot","DuckAssistBot","Meta-ExternalAgent","ClaudeBot","GPTBot","Meta-ExternalFetcher","PerplexityBot"]}
{"domain":"usf.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","AhrefsBot","AhrefsSiteAudit","Twitterbot"]}
{"domain":"kit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"azure-dns.info","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ebay.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":15,"stanza_tokens":["*","bingbot","AdsBot-Google","AdIdxBot","Twitterbot","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","PerplexityBot","anthropic-ai","AmazonBot","meta-externalagent","GPTBot","Applebot-Extended"]}
{"domain":"ualberta.ca","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"maticooads.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_WRONG_SIGNATURE_TYPE","attempts":["ERR_SSL_WRONG_SIGNATURE_TYPE","ERR_SSL_WRONG_SIGNATURE_TYPE"]}
{"domain":"samsungapis.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cctld.us","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nber.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"myworkdayjobs.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ENOTFOUND"]}
{"domain":"ov1o.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"giantpanda.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","PerplexityBot","ClaudeBot","anthropic-ai","Applebot-Extended","Google-Extended"]}
{"domain":"google.com.pa","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"zappos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"reverbnation.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"realme.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fastlylb.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dish.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Lucidworks","*"]}
{"domain":"braze-images.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"nessus.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"boutell.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["ia_archiver"]}
{"domain":"google.com.gt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"signifyd.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thunderbird.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"sophosxl.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"detik.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Googlebot","facebookexternalhit","*"]}
{"domain":"ble.ir","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sliide.cloud","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","ENOTFOUND"]}
{"domain":"netgate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sacdnssedge.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pstatp.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"foreignpolicy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User"],"open":[],"path_restricted":["Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":32,"stanza_tokens":["*","DuckAssistBot","Gemini-Deep-Research","Meta-ExternalFetcher","MistralAI-User","ChatGPT-User","PerplexityBot","Perplexity‑User","Claude-User","AI2Bot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","cohere-training-data-crawler","Cotoyogi","Datenbank Crawler","Diffbot","FacebookBot","Factset_spyderbot","Google-Extended","GPTBot","ICC-Crawler","Kangaroo Bot","Meta-ExternalAgent","netEstate Imprint Crawler","omgili","PanguBot","PetalBot","SemrushBot-OCOB","Timpibot","Webzio-Extended"]}
{"domain":"google.com.ec","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"gitea.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":9,"stanza_tokens":["*","OAI-SearchBot","CCBot","PerplexityBot","anthropic-ai","Claude-Web","ClaudeBot","GPTBot","ChatGPT-User"]}
{"domain":"uniview.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tomsguide.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","AmazonAdBot","bytespider","mistralai","cohere","ai2bot","youbot","omgili","diffbot","kangaroo","img2dataset","amazonbot","amazon-qbusiness","meta-externalagent","meta-webindexer"]}
{"domain":"swr.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":21,"stanza_tokens":["*","applebot-extended","bytespider","ccbot","claudebot","google-extended","cohere-training-data-crawler","deepseekbot","deepseek","deepseek-crawler","diffbot","facebookbot","meta-externalagent","gptbot","grokbot","img2dataset","laiondownloader","mistralai-crawler","omgili","omgilibot","pangubot"]}
{"domain":"china.com.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","TimeoutError"]}
{"domain":"gslbnspk.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"dns-tm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"paho.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","GPTBot","ClaudeBot","Google-Extended","CCBot","anthropic-ai","Bytespider","PetalBot"]}
{"domain":"gazeta.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":276,"stanza_tokens":["*","Googlebot-News","360Spider","80legs","Aboundex","accelobot","Add\\ Catalog","aiHitBot","Alexibot","Aqua_Products","AskJeeves","asterias","awcheckBot","b2w/0.1","BackDoorBot/1.0","BacklinkCrawler","Baiduspider","Barkrowler","BecomeBot","BLEXBot","BlowFish/1.0","Bookmark search tool","BotALot","brandwatch.net","BuiltBotTough","Bullseye/1.0","BunnySlippers","Butterfly","CatchBot","Charlotte","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","Clickagy Intelligence Bot v2","CLIPish","Cliqzbot","COMODO","Comodo-Certificates-Spider","CompSpyBot","Copernic","CopyRightCheck","cosmos","crawler","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","Curious","curl","DataForSeoBot","dataprovider\\.com","DinoPing","discoverybot","DittoSpyder","DomainCrawler","dotbot","dotnetdotcom","Dow\\ Jones\\ Searchbot","dumbot","EasouSpider","EmailCollector","EmailSiphon","EmailWolf","Enterprise_Search","Enterprise_Search/1.0","EroCrawler","es","Exabot","ExtractorPro","EyeMonIT","EzineArticlesLinkScanner","Ezooms","FairAd Client","Feedpresso Content Index Bot","Flaming AttackBot","Foobot","FreeFind","FTRF\\:\\ Friendly","Gaisbot","GetRight/4.2","gigabot","GPTBot","grub","grub-client","Harvest/1.5","Hatena Antenna","hloader","http://linkfluence.com","http://mediacloud.org","http://www.SearchEngineWorld.com  bot","http://www.WebmasterWorld.com  bot","HTTP_Request","HTTP_Request2","httplib","https://developers.snap.com/robots","https://help.qwant.com/bot/","humanlinks","Indy\\ Library","InfoNaviRobot","ip\\-web\\-crawler\\.com","Iron33/1.0.2"]}
{"domain":"homestead.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"lokalise.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"postimg.cc","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"sendsay.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aspiredigital.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"dramaboxdb.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"bankrate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Mediapartners-Google","Googlebot-Image"]}
{"domain":"ubs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.iq","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"google.com.jm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"michaels.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"qh-cdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"EAI_AGAIN","attempts":["EAI_AGAIN","TimeoutError"]}
{"domain":"marthastewart.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"imageshack.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":98,"stanza_tokens":["AddSearchBot","AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","Andibot","anthropic-ai","Applebot","Applebot-Extended","Awario","bedrockbot","bigsur.ai","Brightbot 1.0","Bytespider","CCBot","ChatGPT Agent","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Datenbank Crawler","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","FacebookBot","facebookexternalhit","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","Google-Firebase","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","LinerBot","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","meta-webindexer","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NovaAct","OAI-SearchBot","omgili","omgilibot","OpenAI","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","ShapBot","Sidetrade indexer bot","TerraCotta","Thinkbot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","wpbot","YaK","YandexAdditional","YandexAdditionalBot","YouBot","*"]}
{"domain":"value-domain.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thesaurus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":7,"stanza_tokens":["GPTBot","CCBot","deepcrawl","seerbot/1.0","Applebot-Extended","*","AmazonAdBot"]}
{"domain":"barrons.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":73,"stanza_tokens":["*","googlebot","googlebot-image","GoogleOther","Googlebot-Video","Google-InspectionTool","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","Storebot-Google","google-read-aloud","mediapartners-google","bingbot","msnbot","bingpreview","slurp","yahoo","yandex","YandexImages","baiduspider","Pinterestbot","Yeti","MojeekBot","360Spider","google-cloudvertexbot","duckduckbot","petalbot","Applebot","AppleNewsBot","flipboard","qwantbot","SeznamBot","yisouspider","proximic","admantx","thetradedesk","outbrain","ias_crawler","AmazonAdBot","pubmatic","smartologybot","parselybot","Screaming Frog SEO Spider","DeepCrawl","AhrefsBot","SemrushBot","SimilarWebBot","SISTRIX","OnCrawl","botify","Chrome-Lighthouse","DJSEOcrawler","ChatGPT-User","GPTBot","OAI-SearchBot","facebookexternalhit","facebot","twitterbot","linkedinbot","telegrambot","snapchat","newrelicpinger","siteimprovebot","uptimerobot","pingdom.com_bot_version_1.4","sentry","schema-validator","googleimageproxy","Iframely","Vocabtracker","EpvzCrawl6194680250","Citoid","ZoteroTranslationServer"]}
{"domain":"southernliving.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"premierleague.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sketchup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lycos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","BadBot","EvilScraper","MJ12bot","AhrefsBot"]}
{"domain":"mozaws.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"msnbc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","anthropic-ai"],"open":[],"path_restricted":["Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":51,"stanza_tokens":["*","GPTBot","PerplexityBot","Amazonbot","anthropic-ai","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","HuggingFaceBot","ChatGPT-User","ClaudeBot","Claude-Web","DataForSeoBot","Diffbot","FacebookBot","Google-Extended","Google-NotebookLM","Google-Agent","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","Scrapy","TurnitinBot","cohere-ai","Applebot-Extended","Meta-ExternalAgent","meta-externalagent","OAI-SearchBot","Claude-User","polymarket-news-bot","KalshiBot","polymarket-bot","OpenClaw","gold-ai-event-bridge","TradingBot","trading-pipeline","kalshi-bot","BivariateKalshiPipeline","Falcon Trading Platform","StockMarketAnalyser","FinanceApp","polyclaw-ingester","IsoxyTrader","Neo Trading Bot","MerlinChartWizard","investment-dashboard"]}
{"domain":"geodema.network","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"belkin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":21,"stanza_tokens":["*","Amazonbot","Googlebot-Image","Mediapartners-Google","Storebot-Google","AdsBot-Google","Baiduspider","360Spider","Yisouspider","PetalBot","Bytespider","Sogou web spider","Sogou inst spider","Robozilla","yahoo-mmcrawler","psbot","asterias","yahoo-blogs/v3.9","Yandex","BLEXBot","Nutch"]}
{"domain":"rome2rio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Mediapartners-Google","AdsBot-Google-Mobile","AdsBot-Google","Slurp","Baiduspider"]}
{"domain":"xlivrdr.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ably.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uu.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wsu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"livestream.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"filezilla-project.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nhentai.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"arcticwolf.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"johnlewis.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wandera.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"ustream.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"trulia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["ActiveComplyBot","CriteoBot","GumGumBot","TaraGroup Intelligent Bot","*"]}
{"domain":"sports.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"goldapple.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":44,"stanza_tokens":["*","Googlebot","seznambot","SemrushBot","LinkpadBot","BLEXBot","FlipboardProxy","aiHitBot","trovitBot","grapeshot","MegaIndex","MJ12bot","Detectify","Riddler","DotBot","DISCo Pump","Wget","Teleport Pro","psbot","Twiceler","NetinfoBot","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","grub-client","NPBot","WebReaper"]}
{"domain":"srf.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"instrumentidlyadoma.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"volcfcdndvs1.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hgtv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":8,"stanza_tokens":["*","Mediapartners-Google","Pinterest","GPTBot","ChatGPT-User","CCBot","ClaudeBot","PerplexityBot"]}
{"domain":"msdn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"recruit.co.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE",403]}
{"domain":"dazn.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"gitlab.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"wondershare.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mac.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"rokt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"drpc.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","Claude-Web","PerplexityBot","Perplexity-User","ClaudeBot","Claude-SearchBot","anthropic-ai","Google-Extended"]}
{"domain":"myfitnesspal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":[],"path_restricted":["GPTBot","OAI-SearchBot"]}],"stanza_token_count":2,"stanza_tokens":["ChatGPT-User","*"]}
{"domain":"vrt.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":21,"stanza_tokens":["*","AI2Bot","Amazonbot","Applebot-Extended","anthropic-ai","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","Diffbot","DuckAssistBot","FacebookBot","Google-Extended","GPTBot","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","PerplexityBot","PerplexityUser"]}
{"domain":"lxdns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ngxcld.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"paylocity.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"asd.pics","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"20minutes.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":91,"stanza_tokens":["*","grapeshot","ChatGPT-User","OAI-SearchBot","MistralAI-User","GPTBot","CCBot","Google-Extended","PerplexityBot","anthropic-ai","Claude-Web","ClaudeBot","cohere-ai","Applebot-Extended","MistralAI-Index","FirstRain","rogerbot","AhrefsBot","SearchmetricsBot","MJ12bot","dotbot","Nutch","trendictionbot","XoviBot","Cliqzbot","GnowitNewsbot","MaxPointCrawler","Meltawer","Digimind","Knowings","Sindup","Cision","Talkwater","TurnitinBot","ConveraCrawler","QuepasaCreep","Jetbot","NewsNow","kbcrawl","AmiSoftware","Newzbin","Ask n read","Qwam content intelligence","Zite","Youmag","Synthesio","trendybuzz","scoop.it","linkfluence","5emeRue","Augure","Corporama","readability.com","grub-client","ia_archiver","ia_archiver-web.archive.org","k2spider","libwww","wget","5erue","adequat","adequat-systems","auramundi","coexel","leadbox","mention","Moreover","mytwip","opinion-tracker","proxem","score3","trendeo","vecteurplus","verticalsearch","vsw","winello","Fetch","infoseek","MSIECrawler","Offline Explorer","sitecheck.internetseer.com","SiteSnagger","Teleport","TeleportPro","WebCopier","WebStripper","Zealbot","asknread.com","ellisphere","spotter","Riddler"]}
{"domain":"eldiario.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wampserver.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"biglobe.ne.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"pku.edu.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",404]}
{"domain":"dns.eu","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vwapps.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"victoriassecret.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"india.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","ClaudeBot","ChatGPT","GPTBot","GPT","PerplexityBot","Perplexity-User","claude-web","Google-Extended","ChatGPT-User","SemrushBot","Yandex"]}
{"domain":"colostate.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"redditspace.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ebsco.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","AhrefsBot","MJ12bot","DotBot","Scrapy","Hatena","Grub","ia_archiver"]}
{"domain":"netlify.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","Claude-Web","ClaudeBot","Google-Extended","PerplexityBot"]}
{"domain":"libpng.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"archdaily.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AhrefsBot","msnbot"]}
{"domain":"etrade.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"robertsspaceindustries.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oxfordjournals.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"global-e.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"terabox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["GPTBot","PerplexityBot","ClaudeBot","Google-Extended","*"]}
{"domain":"tistory.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"aiv-cdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"the-sun.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":49,"stanza_tokens":["*","Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News","Google-InspectionTool","GoogleOther","APIs-Google","Mediapartners-Google","DuckDuckBot","Slurp","Twitterbot","Bingbot","adidxbot","MicrosoftPreview","Applebot","AdsBot-Google","AmazonAdBot","facebookexternalhit","Yahoo Ad Monitoring","parse.ly scraper","Screaming Frog SEO Spider","SemrushBot","Botify","OnCrawl","Chrome-Lighthouse","Slackbot-LinkExpanding","Slack-ImgProxy","Slackbot","Snap URL Preview Service","proximic","GumGum-Bot","TTD-Content","Pubmatic","ias_crawler","outbrain","Leikibot","weborama-fetcher","AHC/2.1","SirdataBot","peer39_crawler/1.0","EchoboxBot/1.0","CriteoBot/0.1","AdDefend Site Context Crawler/1.0","SmartologyBot","Dianomi","ChatGPT-User","GPTBot","OAI-SearchBot"]}
{"domain":"ebay.com.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":15,"stanza_tokens":["*","bingbot","AdsBot-Google","AdIdxBot","Twitterbot","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","PerplexityBot","anthropic-ai","AmazonBot","meta-externalagent","GPTBot","Applebot-Extended"]}
{"domain":"leetcode.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"studocu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":19,"stanza_tokens":["*","GPTBot","Google-Extended","YandexBot","Baiduspider","Amazonbot","Meta-ExternalAgent","FacebookBot","Applebot-extended","CCBot","anthropic-ai","ClaudeBot","Claude-web","omgili","cohere-ai","Scrapy","Diffbot","ImagesiftBot","TurnitinBot"]}
{"domain":"gtimg.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"localto.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENETUNREACH"]}
{"domain":"reclameaqui.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"htsc.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"bittorrent.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","GPTBot","ClaudeBot","PerplexityBot","Google-Extended"]}
{"domain":"searchenginejournal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","AwarioRssBot","AwarioSmartBot","Omgilibot","Omgili","proximic","peer39_crawler","peer39_crawler/1.0","Screaming Frog SEO Spider","SiteCheckerBotCrawler","AhrefsBot","sentibot","MJ12bot","BLEXBot"]}
{"domain":"pa.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"concursolutions.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sway.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jumpjump.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"ap.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","GPTBot","Google-Extended","ClaudeBot","PerplexityBot","Amazonbot","bingbot","GnowitNewsbot","ChatGPT-User","OAI-SearchBot","SemrushBot","Applebot"]}
{"domain":"lastampa.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":101,"stanza_tokens":["*","PetalBot","Yandex","PerplexityBot","Amazonbot","AhrefsBot","archive.org_bot","ArchiveBot","Arquivo-web-crawler","AwarioRssBot","AwarioSmartBot","BacklinkCrawler","BIXOCRAWLER","BLEXBot","Bytespider","Cliqzbot","ConveraCrawler","crawler4j","CrystalSemanticsBot","DataForSeoBot","Diffbot","discobot","discoverybot","dloader(NaverRobot)","dotbot","Download Ninja","DuckAssistBot","europarchive.org","Exabot","ExtractorPro","Fetch","Flamingo_SearchEngine","FriendlyCrawler","Gigabot","GumGum Bot","ia_archiver","ia_archiver-web.archive.org","ImagesiftBot","IstellaBot","Jetbot","JikeSpider","Jyxobot","k2spider","Kangaroo Bot","kbcrawl","LexxeBot/1.0","LinkArchiver","LinkextractorPro","Livelapbot","magpie-crawler","meta-externalfetcher","MJ12bot","Moreoverbot","MSIECrawler","nabot","NaverBot","NerdByNature.Bot","netEstate NE Crawler","nNetSeer crawler","NextGenSearchBot","Nicecrawler","NPBot","nutch","Openbot","PanguBot","peer39_crawler","peer39_crawler/1.0","PiplBot","Pixray-Seeker","Primalbot","psbot","QuerySeekerSpider","Quora-Bot","rogerbot","Scrapy","searchpreview","SemrushBot","SEOENGBot","SEOkicks-Robot","SiteBot","SiteBot/0.1","Slurp","Sosospider","spbot","Timpibot","trendictionbot","trovitBot","True_Robot","TurnitinBot","UbiCrawler","uMBot-LN","URL_Spider_Pro","verticalsearch","WBSearchBot","web-archive-net.com.bot","WeSEE:Search","Wotbot","YouBot","Zealbot","AppleBot"]}
{"domain":"gist.build","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"patch.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":36,"stanza_tokens":["CCBot","CCBot/2.0","CCBot/2.0 (http://commoncrawl.org/faq/)","anthropic-ai","cohere-ai","WikiDo","fr_crawler","Yandex","Baiduspider","Baiduspider-image","Baiduspider-video","Baiduspider-favo","Baiduspider-news","Baiduspider-cpro","Baiduspider-ads","trendictionbot","Bitvorebot","BLP_bbot","heritrix","magpie-crawler","Kraken","moatbot","bhcBot","SemrushBot","Synthesio","AhrefsBot","BrandONbot","GermCrawler","Sogou","Exabot","MaxPointCrawler","ADmantX","Slurp","*","Twitterbot","AmazonAdBot"]}
{"domain":"a1q7.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"srv53.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"kandji.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"liveinternet.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google*","*"]}
{"domain":"sbixby.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"business-standard.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"fujitsu.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"sibforms.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"seagate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AEM Seagate","HypeStat","EtaoSpider","AppCodesCrawler","bingbot","*"]}
{"domain":"paycomonline.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cargurus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["trovitBot","Mediapartners-Google","AmazonAdBot","*","Googlebot"]}
{"domain":"heytapdownload.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"doxygen.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"worldfcdn2.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"carnival.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"packagist.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"apkpure.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"svt.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":58,"stanza_tokens":["*","Bingbot","Googlebot","ChatGPT-User","OAI-SearchBot","Claude-User","Claude-SearchBot","PerplexityBot","MistralAI-User","facebookexternalhit","LinkedInBot","Twitterbot","TikTokSpider","anthropic-ai","ClaudeBot","Bravebot","GPTBot","Google-Extended","Applebot-Extended","Meta-ExternalAgent","Meta-ExternalFetcher","FacebookBot","Amazonbot","cohere-ai","cohere-training-data-crawler","Diffbot","AI2Bot","AI2Bot-Dolma","CCBot","BaiChuanBot","Bytespider","DeepSeekBot","GrokBot","xAI-Grok","Grok-DeepSearch","TansuoBot","WeChatBot","Yibot","ZhipuAI","360Spider","Sogoubot","PanguBot","PetalBot","YandexAdditional","YandexAdditionalBot","Omgilibot","Omgili","Timpibot","VelenPublicWebCrawler","Webzio-Extended","ImagesiftBot","img2dataset","FriendlyCrawler","ICC-Crawler","Kangaroo+Bot","SemrushBot-OCOB","iaskspider/2.0","YouBot"]}
{"domain":"npmjs.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"jivox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adweek.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":93,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","Googlebot","AdsBot-Google","Googlebot-Image","Twitterbot","Baiduspider","008","AdsTxtCrawler","AI2Bot","Ai2Bot-Dolma","anthropic-ai","Aranet-SearchBot","Archive-It","archive.org_bot","Arquivo-web-crawler","Authory","bl.uk_lddc_bot","bne.es_bot","bnf.fr_bot","ChatGLM-Spider","Claude-Web","CloudVertexBot","cohere-ai","cohere-training-data-crawler","CohereBot","Cotoyogi","Crawl4AI","CUCH.org Archive Bot","Dataprovider.com","Datenbank Crawler","dcrawl","DeepSeekBot","FacebookBot","GoogleOther","HelloworkJobPostingBot","heritrix","HTTrack","HTTrack 3.0","ia_archiver","ia_archiver-web.archive.org","IABot","iAskBot","iaskspider","ICC-Crawler","imageSpider","IndeedJobBot","Internet Archive","Kangaroo Bot","KimiBot","KrawlerBot","KunatoCrawler","laion-huggingface-processor","LCC","MetaInspector","MetaJobBot","mirrorweb","MistralBot","netarkivindsamling","netEstate Imprint Crawler","newspaper","Nicecrawler","Nutch","nyt_scraping","Offline Explorer","omgili","OpenindexSpider","PanguBot","Potions","SBIntuitionsBot","Scrapy","ServerHunterSpider","SmarshBot","special_archiver","Spider","StatsDroneBot","Thinkbot","Timpibot","Turnitin","VelenPublicWebCrawler","webzio-extended","WRTNBot","XY-Archive-Compliance","XY-Archive-Compliance-Archiver","ZenphonyBot"]}
{"domain":"cbslocal.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"aliexpress.us","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"worldometers.info","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pin.it","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","ENOTFOUND",200]}
{"domain":"meituan.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"prweb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"os-lb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"loxonecloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sb-cd.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mobiuspace.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"blogspot.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"capcutapi.us","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"protrafficinspector.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"byu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nsf.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"itsupport247.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"skyscanner.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["Mediapartners-Google","YisouSpider","Leikibot","ShopWiki","NPBot","psbot","HenryTheMiragoRobot","gigabot","appie","Meta-ExternalAgent","AhrefsBot","*"]}
{"domain":"awsdns-19.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ttcdn-us.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rediff.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"qvc.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":418,"error":null,"attempts":[418,418]}
{"domain":"lovable.dev","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","PerplexityBot","Google-Extended","Applebot-Extended"]}
{"domain":"srmdata.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"inaccuracy.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"ucsc.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":26,"stanza_tokens":["*","GPTBot","ClaudeBot","anthropic-ai","Google-Extended","CCBot","Bytespider","Applebot-Extended","Meta-ExternalAgent","cohere-ai","ImagesiftBot","Omgilibot","AhrefsBot","SemrushBot","MJ12bot","DotBot","PetalBot","BLEXBot","DataForSeoBot","SerpstatBot","ZoominfoBot","Barkrowler","magpie-crawler","MauiBot","SeekportBot","Nimbostratus-Bot"]}
{"domain":"greenhouse.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yeastar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"theregister.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":68,"stanza_tokens":["Googlebot","Bingbot","Slurp","DuckDuckBot","Baiduspider","YandexBot","Applebot","OAI-SearchBot","ChatGPT-User","PerplexityBot","Claude-User","Claude-SearchBot","claude-code","AdsBot-Google","AdsBot-Google-Mobile","Mediapartners-Google","facebookexternalhit","Twitterbot","LinkedInBot","Pinterestbot","Slackbot","Slackbot-LinkExpanding","Discordbot","TelegramBot","WhatsApp","ias_crawler","ias_wombles","ias_admantx","DoubleVerifyBot","DVBot","proximic","peer39_crawler","GrapeshotCrawler","grapeshot","GumGum","GlancelotBot","TTD-Content","PubMatic Crawler Bot","SinceraSyntheticUser","AmazonAdBot","Leikibot","SlickBot","GPTBot","ClaudeBot","anthropic-ai","Claude-Web","Google-Extended","Google-CloudVertexBot","Meta-ExternalAgent","Meta-ExternalFetcher","FacebookBot","Applebot-Extended","Amazonbot","Bytespider","CCBot","cohere-ai","Diffbot","YouBot","DuckAssistBot","Timpibot","Omgilibot","Omgili","ImagesiftBot","GrokBot","AI2Bot","FirecrawlAgent","Scrapy","*"]}
{"domain":"tubi.video","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.com.py","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"rapidssl.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bip.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adspend.space","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"mo.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"coxautoinc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tubemogul.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"krxd.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nerdwallet.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"rusprofile.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":13,"stanza_tokens":["Mediapartners-Google","*","YandexBot","msnbot","GPTBot","ChatGPT-User","OAI-SearchBot","Google-Extended","ClaudeBot","CCBot","PerplexityBot","DeepSeekBot","bingbot"]}
{"domain":"queniuiq.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"oaistatsig.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"audiomack.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"imolive.tv","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"newgrounds.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"siteground.us","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"memcached.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"hismarttv.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"acint.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"justwatch.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sqlite.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kwimgs.com","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":["CERT_HAS_EXPIRED",400]}
{"domain":"nd.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dutils.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"kaltura.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"uq.edu.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"estadao.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":11,"stanza_tokens":["*","GPTBot","ChatGPT-User","CCBot","Bytespider","Google-Extended","OAI-SearchBot","ClaudeBot","PerplexityBot","anthropic-ai","PetalBot"]}
{"domain":"siemens.de","fetch_ok":false,"usable":false,"reason":"other","status":405,"error":null,"attempts":[405,405]}
{"domain":"uadns.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"nationalarchives.gov.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Googlebot","Bingbot","Slurp","DuckDuckBot","*"]}
{"domain":"yna.co.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":34,"stanza_tokens":["*","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","ChatGPT-User","OAI-SearchBot","CCBot","anthropic-ai","omgili","omgilibot","FacebookBot","Amazonbot","PerplexityBot","YouBot","Applebot-Extended","Bytespider","ClaudeBot","Diffbot","Meta-ExternalAgent","Claude-Web","cohere-ai","Ai2Bot","Ai2Bot-Dolma","FriendlyCrawler","ICC-Crawler","ImagesiftBot","PetalBot","Scrapy","Timpibot","VelenPublicWebCrawler","Webzio-Extended","DeepSeekBot"]}
{"domain":"ow.ly","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"tfkc.de","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"EAI_AGAIN","attempts":["EAI_AGAIN","EAI_AGAIN"]}
{"domain":"yandex.by","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","yandex","Twitterbot"]}
{"domain":"cdnhwcuim119.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"ebaystatic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"aiscore.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"mbrd.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"nocookie.net","fetch_ok":false,"usable":false,"reason":"other","status":530,"error":null,"attempts":["ENOTFOUND",530,"TimeoutError"]}
{"domain":"lazybumblebee.com","fetch_ok":false,"usable":false,"reason":"other","status":521,"error":null,"attempts":["ENOTFOUND",521,"TimeoutError"]}
{"domain":"kaggle.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"ollama.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"boredpanda.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["facebookexternalhit","Bytespider","Baiduspider","PetalBot","TikTokSpider","Twitterbot","Bingbot","*"]}
{"domain":"eatingwell.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"mercadopago.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"hosteurope.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dmm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yango.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["Googlebot","Yandex","GPTBot","ClaudeBot","PerplexityBot","CCBot","Google-Extended","Bingbot","Applebot","Amazonbot","*"]}
{"domain":"feednews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","Google-Display-Ads-Bot","*"]}
{"domain":"nationalacademies.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"queniusy.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"gazprombank.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sicredi.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"myqnapcloud.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"parade.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ct.gov","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"spacex.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"atlasobscura.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":3,"stanza_tokens":["Node/simplecrawler","*","GPTBot"]}
{"domain":"adspower.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"samsung-gamelauncher.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ocn.ne.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"hashicorp.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":[429,"TimeoutError",429]}
{"domain":"pcre.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jivo.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"tanium.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rg.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":4,"stanza_tokens":["Yandex","Googlebot","GPTBot","*"]}
{"domain":"bpldns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"swiggy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jiomart.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"tclking.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"anticheatexpert.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awstrack.me","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"icio.us","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"bdurl.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mailgun.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"alldatasheet.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"zetaglobal.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tinkercad.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mobalytics.gg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yandexadexchange.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"collider.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","\"008\"","anthropic-ai","Applebot","Applebot-extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","GPTBot","HTTrack","Nutch","Offline Explorer","omgili","Scrapy","YouBot"]}
{"domain":"peatix.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["dotbot","*"]}
{"domain":"turnitin.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"arte.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pbase.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"bible.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ns.ph","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"skcdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"polygon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":20,"stanza_tokens":["*","GPTBot","Google-Extended","Applebot-Extended","Applebot","AmazonAdBot","anthropic-ai","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","Diffbot","FacebookBot","ImagesiftBot","Omgilibot","Omgili","PerplexityBot","YouBot","Googlebot-News"]}
{"domain":"google.com.uy","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"ama-assn.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"viglink.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"kobo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bsgslb.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"dhgate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":86,"stanza_tokens":["*","Googlebot","YandexBot","bingbot","OAI-SearchBot","ChatGPT-User","GPTBot","ia_archiver","NaverBot","Yeti","Googlebot-Image","BaiduSpider","Sosospider","HaosouSpider","Sogou web spider","Sogou inst spider","Sogou spider","Sogou wap spider","almaden","ASPSeek","Axmo","BecomeBot","CherryPicker","Crescent Internet ToolPak","DISCo Pump","Download Ninja","DTS Agent","EmailCollector","EmailSiphon","EmailWolf","Expired Domain Sleuth","Franklin Locator","Gaisbot","grub","htdig","HTTrack","iaea.org","IconSurf","Iltrovatore-Setaccio","Indy Library","InternetSeer.com","IUPUI","larbin","libwww","LNSpiderGuy","lwp-trivial","MetaTagRobot","Missigua Locator","mozDex","MSIECrawler","NextGenSearch","NPbot","Nutch","Offline Explorer","Oracle Ultra Search","PictureOfInternet","PlantyNet","psbot","QuepasaCreep","RPT-HTTPClient","sitecheck.internetseer.com","SiteSnagger","spider.acont.de","Sqworm","SSM Agent","szukacz","TAMU","Teleport","TeleportPro","Telesoft","TurnitinBot","TutorGig","Ultraseek","WebCopier","WebEMailExtractor","WebReaper","Webster Pro","WebStripper","WebZIP","Wotbox","Xenu","Zao","Zealbot","ZipppBot","ZyBORG","AdsBot-Google"]}
{"domain":"config-security.com","fetch_ok":false,"usable":false,"reason":"other","status":525,"error":null,"attempts":["ECONNRESET",525,525]}
{"domain":"hessen.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","BUbiNG","SemrushBot","SemrushBot-SA"]}
{"domain":"tplinkra.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"tradedoubler.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"myoas.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNREFUSED"]}
{"domain":"runescape.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nla.gov.au","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"alikunlun.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ntp-fireos.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ETIMEDOUT"]}
{"domain":"bxcdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wsimg.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"thetrainline.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AdsBot-Google","AdsBot-Google-Mobile"]}
{"domain":"spec.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mercadopago.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Googlebot-Image","Mediapartners-Google","Adsbot-Google","Twitterbot"]}
{"domain":"beyondtrustcloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bytelb.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"queniukw.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"queniuum.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"latamairlines.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"biorxiv.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":43,"stanza_tokens":["BLEXBot","Bytespider","Exabot","Scanbot","Semrush","SemrushBot","YoudaoBot","Baiduspider","YandexBot","openai","Petalbot","Bolt","BunnySlippers","ChatGPT-User","GPTBot","Gigabot","HybridBot","JikeSpider","SMTBot","ScreenerBot","SiteLockSpider","SuperBot","SuperHTTP","Amazonbot","AhrefsBot","Applebot","SEMrush","DataForSeoBot","ClaudeBot","DotBot","phxbot","AwarioBot","SeekportBot","MJ12bot","Majestic12","facebookexternalhit/1.1","PerplexityBot","Qwantbot","CCBot","python-requests/2.32.3","wget","curl","*"]}
{"domain":"txlivecdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"ev2-msedge.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"strip2.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bspb.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"ospserver.net","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":["TimeoutError",503,503]}
{"domain":"qq.com.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ndcpp.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bangcdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cdn28.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"grammarly.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"qihucdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"farlightgames.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nic.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vivo.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"10086.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ERR_SSL_UNSAFE_LEGACY_RENEGOTIATION_DISABLED",404]}
{"domain":"google.co.ma","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"gldn.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"everydayhealth.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":109,"stanza_tokens":["*","\"008\"","AddSearchBot","AI2Bot","Ai2Bot-DeepResearchEval","Ai2Bot-Dolma","Amazonbot","AmazonBuyForMe","Amzn-SearchBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot","Applebot-extended","atlassian-bot","AutoRAG","AwarioSmartBot","AzureAI-SearchBot","Big Sur AI","bigsur.ai","Brandwatch","Bravebot","Brightbot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","Cloudflare-AI-Search","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Devin","Diffbot","Direqt Anomura","DuckAssistBot","Echobot Bot","ExaBot","FacebookBot","Factset_spyderbot","FirecrawlAgent","GPTBot","GROK-2","HTTrack","iAsk","iAskBot","iaskspider","ICC Crawler","ICC-Crawler","ImageSiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","Kunato","laion-huggingface-processor","LCC","LINER Bot","LinerBot","LinkupBot","Manus-User","Meta-ExternalAgent","Meta-ExternalFetcher","meta-webindexer","mistral.ai","MistralAI-User","netEstate Imprint Crawler","NovaAct","Novellum AI Crawl","Nutch","OAI-SearchBot","Offline Explorer","omgili","opencode","PanguBot","PeopleInc-DCipher-Scraper/1.1.0","Perplexity-User","PerplexityBot","PetalBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","Scrapy","SeekrBot","SemrushBot-OCOB","SemrushBotSwa","ShapBot","Spider","TavilyBot","TerraCotta","Timpibot","Trae","TwinAgent"]}
{"domain":"warthunder.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xda-developers.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","\"008\"","anthropic-ai","Applebot","Applebot-extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","GPTBot","HTTrack","Nutch","Offline Explorer","omgili","Scrapy","YouBot"]}
{"domain":"br.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"gzip.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tw1.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UND_ERR_HEADERS_OVERFLOW","attempts":["UND_ERR_HEADERS_OVERFLOW","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"mrktmtrcs.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"marketo.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"rutarget.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pardot.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bfmio.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"zdnscloud.biz","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"gohugo.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"flixcdn.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"adtng.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"browser-intake-datadoghq.eu","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"huaweicloud-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rice.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zdnscloud.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"farpost.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"litres.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"meganameservers.eu","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"youboranqs01.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"palm.tech","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"tnt-ea.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"x.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","GPTBot","ChatGPT-User","PerplexityBot","ClaudeBot","Google-Extended","Applebot-Extended"]}
{"domain":"firstbytedns.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"theknot.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"aaa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bytegecko.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"thejeu.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"chapturist.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"homes.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"dyn.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"desync.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["ia_archiver"]}
{"domain":"strln.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"enlightenment.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["Googlebot","bingbot","Baiduspider","DuckDuckBot","YandexBot","Yeti","Applebot","*"]}
{"domain":"infogram.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tripcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mavrtracktor.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"dns-servers.vn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"harpersbazaar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AmazonAdBot","Google-InspectionTool","ChatGPT-User","GPTBot","OAI-SearchBot","*"]}
{"domain":"ertelecom.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"zohopublic.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"casio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":638,"stanza_tokens":["01h4x.com","360Spider","404enemy","80legs","ADmantX","AIBOT","ALittle Client","ASPSeek","Abonti","Aboundex","Aboundexbot","Acunetix","AfD-Verbotsverfahren","AhrefsBot","AhrefsSiteAudit","AiHitBot","Aipbot","Alexibot","AllSubmitter","Alligator","AlphaBot","Anarchie","Anarchy","Anarchy99","Ankit","Anthill","Apexoo","Aspiegel","Asterias","Atomseobot","Attach","AwarioRssBot","AwarioSmartBot","BBBike","BDCbot","BDFetch","BLEXBot","BackDoorBot","BackStreet","BackWeb","Backlink-Ceck","BacklinkCrawler","Badass","Bandit","Barkrowler","BatchFTP","Battleztar Bazinga","BetaBot","Bigfoot","Bitacle","Black Hole","BlackWidow","Blackboard","Blow","BlowFish","Boardreader","Bolt","BotALot","Brandprotect","Brandwatch","Buck","Buddy","BuiltBotTough","BuiltWith","Bullseye","BunnySlippers","BuzzSumo","CATExplorador","CCBot","CODE87","CSHttp","Calculon","CazoodleBot","Cegbfeieh","CensysInspect","CheTeam","CheeseBot","CherryPicker","ChinaClaw","Chlooe","Citoid","Claritybot","Cliqzbot","Cloud mapping","Cocolyzebot","Cogentbot","Collector","Copier","CopyRightCheck","Copyscape","Cosmos","Craftbot","Crawling at Home Project","CrazyWebCrawler","Crescent","CrunchBot","Curious","Custo","CyotekWebCopy","DBLBot"]}
{"domain":"dyndns.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"gq.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","FacebookExternalHit","Twitterbot","Pinterestbot","LinkedInBot","TikTokSpider","Storebot-Google","Google-InspectionTool","AmazonAdBot","Google-Extended","Google-CloudVertexBot","GoogleOther","Applebot-Extended","Amazonbot","meta-webindexer","meta-externalagent","meta-externalfetcher","ClaudeBot","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","cohere-training-data-crawler","cohere-ai","CCBot","PanguBot","PetalBot","Bytespider","Diffbot","DuckAssistBot","MistralAI-User","Timpibot","Webzio","Webzio-Extended","archive.org_bot","ia_archiver","ia_archiver-web.archive.org","heritrix"]}
{"domain":"telenet.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nrw.de","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"gamestop.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"wizzair.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"braintreegateway.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pbxai.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"ecombdapi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gdms.cloud","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"unicode.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","AhrefsBot","Baiduspider","CCBot","Gigabot","betaBot","dotbot","MJ12bot","msnbot","BLEXBot","SeznamBot","Viber/18.3.2.0"]}
{"domain":"picpay.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"viator.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"glbimg.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nashjournal.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"neon.tech","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"twingate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"msu.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","*"]}
{"domain":"jiveip.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"disney.co.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["CERT_HAS_EXPIRED",403]}
{"domain":"azure.us","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"oppo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eleconomista.es","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"aviasales.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gfx.ms","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"eonline.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":25,"stanza_tokens":["Amazonbot","anthropic-ai","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","DataForSeoBot","Diffbot","FacebookBot","Google-Extended","GPTBot","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Scrapy","TurnitinBot","*"]}
{"domain":"doh.pub","fetch_ok":false,"usable":false,"reason":"other","status":502,"error":null,"attempts":[502,"TimeoutError",502]}
{"domain":"westernunion.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ntv.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Google-CloudVertexBot","Googlebot","*"]}
{"domain":"uu.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"uni-heidelberg.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rixengine.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"queniuso.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"ew.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"nimo.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Applebot-Extended","Applebot"]}
{"domain":"washingtontimes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["YandexBot","Spinn3r","Mail.RU_Bot","Baiduspider","Slurp","AhrefsBot","bingbot","msnbot","CCBot","omgilibot","omgili","GoogleProducer","Amazonbot","applebot","*","googlebot"]}
{"domain":"hindawi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":6,"stanza_tokens":["*","facebookexternalhit","LinkedInBot","Twitterbot","GPTBot","Google-Extended"]}
{"domain":"travelandleisure.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"cnki.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zomato.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"edgetcdn.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"apple-mapkit.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"omniroot.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","EHOSTUNREACH"]}
{"domain":"ifconfig.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"plesk.page","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vcmdiawe.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",200]}
{"domain":"drugs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":22,"stanza_tokens":["Mediapartners-Google","*","008","AI2Bot","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","Diffbot","EtaoSpider","Meta-ExternalAgent","mistral.ai","PerplexityBot","SeekrBot","Timpibot","Webzio-Extended","Wrtn","xAI-Web-Crawler"]}
{"domain":"pubnub.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"au.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xmlsoft.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"kadencewp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"verisign.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adzerk.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"aepd.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eufylife.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["Googlebot","Googlebot-image","*","adsbot-google","Nutch","AhrefsBot","AhrefsSiteAudit","MJ12bot","Pinterest"]}
{"domain":"hardened-php.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dolynkcloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kvant-telecom.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UND_ERR_SOCKET","attempts":["UND_ERR_SOCKET","ENOTFOUND"]}
{"domain":"blinkit.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"minecraft-services.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"getgo.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"moovitapp.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"micpn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Twitterbot","google-rcs-conversation"]}
{"domain":"aliapp.org","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","DEPTH_ZERO_SELF_SIGNED_CERT"]}
{"domain":"correoargentino.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"in.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"infoseek.co.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"otzovik.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"openxcdn.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"54.in-addr.arpa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"campaign-archive.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"iol.co.za","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":26,"stanza_tokens":["*","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","GPTBot","omgili","semrushbot","semrushbot-ba","semrushbot-ct","semrushbot-si","semrushbot-swa","serpstatbot","zoombot"]}
{"domain":"pravda.com.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"20min.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":36,"stanza_tokens":["*","Mediapartners-Google","Amazonbot","Bytespider","cohere-ai","CCBot","GPTBot","anthropic-ai","Google-CloudVertexBot","Google-Extended","Applebot-Extended","omgilibot","omgili","ClaudeBot","Claude-Web","Deepseek-V3","DeepSeek-V3","PerplexityBot","Diffbot","OAI-SearchBot","NewsNow","news-please","Meta-ExternalAgent","meta-externalagent","Meta-ExternalFetcher","FacebookBot","magpie-crawler","BLP_bbot","sentibot","peer_39_crawler/1.0","peer39_crawler","dotbot","SEMrushBot","Scrapy","AhrefsBot","DataForSeoBot"]}
{"domain":"transsion-os.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"businessweek.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"openlibrary.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","Baiduspider","Googlebot","AhrefsBot","meta-externalagent","anthropic-ai","ClaudeBot","openai","ChatGPT","bingbot","Yandex","*bot"]}
{"domain":"directadmin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kaiserpermanente.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"trimble.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mercurynews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":68,"stanza_tokens":["*","AI2Bot","Ai2Bot-Dolma","Amazonbot","Amzn-SearchBot","anthropic-ai","Applebot","Applebot-Extended","Archive.org_bot","Baiduspider","Brightbot 1.0","Bytespider","carbon-umbrella-bot","CCBot","ChatGLM-Spider","ChatGPT-User","ClaudeBot","Claude-Searchbot","Claude-User","Claude-Web","Cohere-training-data-crawler","Cotoyogi","Dalvik","Datenbank Crawler","Diffbot","DocSpotBot","DuckAssistBot","FacebookBot","GnowitNewsbot","Google-Extended","GPTBot","grok-crawler","ia_archiver-web.archive.org","ICC-Crawler","imageSpider","Kangaroo Bot","laion-huggingface-processor","LCC","Mantisbot","Meta-ExternalAgent","MistralAI-User","netEstate Imprint Crawler","OAI-SearchBot","omgili","PanguBot","PerplexityBot","Perplexity-User","PetalBot","SBIntuitionsBot","SirdataBot","Spider","Thinkbot","Timpibot","VelenPublicWebCrawler","Vivaldi","Webzio-extended","YandexBot","YouBot","AmazonAdBot","facebookexternalhit","grapeshot","Mozilla/5.0 (compatible; EchoboxBot/1.0; hash/w4mwnpbXf3MFAbxOkJRw; +http://www.echobox.com)","Twitterbot","Mozilla/5.0 (compatible; NewsRoom.BI/0.1; +http://www.newsroom.bi/bot.html)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36(KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; mrfCompass-Marshall/1.0)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Jukebox/1.0)"]}
{"domain":"nlnetlabs.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"expiredwixdomain.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AdsBot-Google-Mobile","AdsBot-Google","PetalBot","dotbot","AhrefsBot"]}
{"domain":"infomaniak.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"videonow.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ana.co.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"getui.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"iranlms.ir","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"tdatamaster.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"proxytg.ink","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"EAI_AGAIN","attempts":["EAI_AGAIN","EAI_AGAIN"]}
{"domain":"zee5.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","AdsBot-Google","Googlebot","Googlebot-Image","PetalBot"]}
{"domain":"adx.ws","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"onlamp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":65,"stanza_tokens":["*","Baiduspider","PetalBot","yisouspider","008","ChatGPT-User","ChatGPT-User/2.0","PerplexityBot","Perplexity-User","DuckAssistBot","Applebot","bingbot","Googlebot","ClaudeBot","claude-web","Claude-SearchBot","YouBot","Applebot-Extended","MistralAI-User","meta-webindexer","GoogleAgent-Mariner","DotBot","archive.org_bot","Pinterestbot","TelegramBot","kakaotalk-scrap","getstream.io/opengraph-bot","RamblerMail","Brave","OAI-SearchBot","GPTBot","anthropic-ai","Google-Extended","cohere-ai","CCBot","AI2Bot","Amazonbot","Amazonbot-Video","Bytespider","meta-externalagent","Diffbot","omgili","TimpiBot","SeznamBot","Exabot","YandexBot","Sogou","360Spider","YisouSpider","Baiduspider-Render/2.0","MJ12bot","DataForSeoBot","AliyunSecBot","ArchiveTeam","ArchiveTeam ArchiveBot","AwarioBot","ZoominfoBot","ali-implementer","Blueno","BIGO-baiguoyuan","WorksOgCrawler","OnPageBot","DoCoMo","SAMSUNG-SGH-E250","TA-Googlebot"]}
{"domain":"samsungotn.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"worldtimeserver.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"netscalergateway.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kwaicdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zoho.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot-Image","zs-crawler","zoho-scrapy"]}
{"domain":"chinaz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aap.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["SiteAuditBot","SemrushBot","YisouSpider","*"]}
{"domain":"yolasite.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"preply.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"cookieadmin.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"midjourney.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"modsecurity.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"meteoblue.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"telegraaf.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["*","Googlebot-News","Amazonbot","anthropic-ai","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","Diffbot","FacebookBot","Google-Extended","GPTBot","magpie-crawler","omgili","omgilibot","PerplexityBot","Google-CloudVertexBot","meta-externalagent","meta-externalfetcher","Ahrefsbot","Archive.org_bot","Bravebot","Claude-Searchbot","Claude-User","DeepSeekBot","Meta-WebIndexer","MistralAI-Index","MistralAI-User","OAI-AdsBot","OAI-Searchbot","Perplexity-User","Youbot"]}
{"domain":"kxcdn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"exitlag.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"yr.no","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":26,"stanza_tokens":["*","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","Diffbot","FacebookBot","Google-Extended","GPTBot","ImagesiftBot","Meta-ExternalAgent","meta-externalagent","NewsNow","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Scrapy"]}
{"domain":"finanzen.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","Flamingo_SearchEngine","*"]}
{"domain":"hyprmx.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vistaprint.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Storebot-Google","/0.1 (+https://www.criteo.com/criteo-crawler/)"]}
{"domain":"telia.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"google.co.ls","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"2miners.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","*"]}
{"domain":"52.in-addr.arpa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"lnkdns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nus.edu.sg","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"namebright.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"zonatelecom.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"superdnsserver.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kommersant.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Yandex","ltx71 - (http://ltx71.com/)","MJ12bot","SemrushBot","TelegramBot"]}
{"domain":"freshworksapi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"trasre.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vc.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zdnscloud.info","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"sumup.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"namefind.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"beeg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"guzelhosting.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"freshworks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"king.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mediagotechnology.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"sellvir.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"iso.ch","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"huya.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["Baiduspider","Googlebot","sogou spider","Sosospider","YodaoBot","Bingbot","Slurp","MSNBot","360Spider","YisouSpider","Chinasospider","*"]}
{"domain":"cnblogs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"royalsocietypublishing.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["008","*"]}
{"domain":"softpedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":33,"stanza_tokens":["Mediapartners-Google","Google-Site-Verification","Adsbot-Google","GoogleOther","AmazonAdBot","*","bingbot","GPTBot","PerplexityBot","Bytespider","Amazonbot","ClaudeBot","CCBot","TweetmemeBot","QuerySeekerSpider","NING","SputnikBot","Cliqzbot","Flamingo_SearchEngine","Vegi bot","demandbase-bot","trendkite-akashic-crawler","DemandbasePublisherAnalyzer","GrapeshotCrawler","YaK","MauiBot","GermCrawler","PaperLiBot","BLEXBot","PetalBot","DataForSeoBot","Buck","SeekportBot"]}
{"domain":"bradesco.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","PerplexityBot","Perplexity-User","Google-Extended","Applebot-Extended"]}
{"domain":"gccdn.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"falconnet.app","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dtscdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"softbank.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"dr.dk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":39,"stanza_tokens":["GPTBot","ChatGPT-user","Google-Extended","Google-NotebookLM","Gemini-Deep-Research","OAI-SearchBot","CCbot","anthropic-ai","Claude-Web","ClaudeBot","Claude-User","Claude-SearchBot","omgili","omgilibot","PerplexityBot","Perplexity-User","DuckAssistBot","DeepSeekBot","DeepSeek","Amazonbot","Amazonbot-Video","Applebot-Extended","Applebot","Bytespider","cohere-ai","Timpibot","YouBot","Brightbot 1.0","FirecrawlAgent","PetalBot","MistralAI-User","VelenPublicWebCrawler","wpbot","iAskBot","PanguBot","meta-externalagent","Ai2Bot-Dolma","ZanistaBot","*"]}
{"domain":"sydney.edu.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["FunnelBack","*"]}
{"domain":"zazzle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Bingbot","facebookexternalhit","Pinterestbot","Twitterbot"]}
{"domain":"jal.co.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"iherb.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"marksandspencer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"virtualbox.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"sndcdn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":15,"stanza_tokens":["anthropic-ai","ClaudeBot","Claude-Web","GPTBot","ChatGPT-User","OAI-SearchBot","CCBot","PerplexityBot","Google-Extended","Applebot-Extended","Bytespider","Amazonbot","Meta-ExternalAgent","cohere-ai","*"]}
{"domain":"drive2.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":19,"stanza_tokens":["*","Applebot","ImagesiftBot","OAI-SearchBot","GPTBot","EzoicBot","PerplexityBot","ClaudeBot","meta-externalagent","meta-externalfetcher","Amazonbot","ShapBot","Googlebot","Mediapartners-Google","ia_archiver","Twitterbot","Yandex","DuckDuckBot","bingbot"]}
{"domain":"mango.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pcgamer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","AmazonAdBot","bytespider","mistralai","cohere","ai2bot","youbot","omgili","diffbot","kangaroo","img2dataset","amazonbot","amazon-qbusiness","meta-externalagent","meta-webindexer"]}
{"domain":"stlouisfed.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","GPTBot","ClaudeBot","PerplexityBot","Google-Extended"]}
{"domain":"youngjoygame.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENETUNREACH"]}
{"domain":"alexa.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"fcomet.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"opensea.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"tabletcaptiveportal.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","ENOTFOUND"]}
{"domain":"khaleejtimes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":8,"stanza_tokens":["facebookexternalhit","meta-externalagent","*","Google-Extended","Gemini-Deep-Research","ClaudeBot","anthropic-ai","AppleBot-Extended"]}
{"domain":"correios.com.br","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"123rf.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":7,"stanza_tokens":["*","GPTBot","Amazonbot","CCBot","anthropic-ai","Claude-Web","Google-Extended"]}
{"domain":"freelancer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["JoobleBot","PetalBot","*","Mediapartners-Google","AdIdxBot","Google-Display-Ads-Bot"]}
{"domain":"acast.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"lloydsbank.co.uk","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"kakaku.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Mediapartners-Google","bingbot","CCBot","Bytespider"]}
{"domain":"aqsi.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Yandex","*","Googlebot-Image","duggmirror"]}
{"domain":"pressreader.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":74,"stanza_tokens":["*","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","BingPreview","Baiduspider","CCBot","ChatGPT-User","Claude-Web","Claude-User","ClaudeBot","Claude-SearchBot","cohere-ai","cohere-training-data-crawler","Diffbot","DuckAssistBot","DeepSeekBot","FacebookBot","FriendlyCrawler","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","Gemini-Deep-Research","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","LinkedInBot","Meta-ExternalAgent","Meta-ExternalFetcher","MistralAI-User","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","Perplexity-User","PetalBot","Scrapy","SemrushBot","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot","YandexBot","facebookexternalhit","Twitterbot","Slackbot-LinkExpanding","Googlebot","Googlebot-News","Googlebot-Image","Googlebot-Mobile","Storebot-Google","AdsBot-Google","AdsBot-Google-Mobile","Bingbot","Yandex","Google-InspectionTool","Google-Site-Verification","Slurp","DuckDuckBot","Naverbot","Bingbot/Render","AdIdxBot","MSNBot-Media","AccessibleWebBot"]}
{"domain":"cloudlinks.cn","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"bilivideo.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wsdvs.info","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"unwomen.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"channelnewsasia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ghcr.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"aps.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"planfix.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"azure-dns.org","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cloudpath.live","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"nag.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ctecdn.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tildacdn.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"sitemaps.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"epam.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"theweathernetwork.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","AmazonAdBot","Hootsuite","HootsuiteBot","OgScrper","OgScraper","Hootsuite-Authoring"]}
{"domain":"eagllwin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"amazon.sg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"youradchoices.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["iSec_Bot","*"]}
{"domain":"govinfo.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dynu.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"transmissionbt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"brdtnet.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"cdnsolutions.media","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["Googlebot","Mediapartners-Google","Yahoo! Slurp","bingbot","Yandex","Baiduspider","Sogou","ia_archiver","IPS-Agent","BLEXBot","Pandalytics","IonCrawl","*"]}
{"domain":"cloud.sap","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Nuclei*","*jndi*","*"]}
{"domain":"aarki.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"capcutstatic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"anker-in.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dominos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"impact.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["trendkite-akashic-crawler","Bytedance","Bytespider","RSiteAuditor","XoviOnpageCrawler","BLEXBot","RuxitSynthetic","8LEGS","*"]}
{"domain":"vinted.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":7,"stanza_tokens":["*","grapeshot","GPTBot","ClaudeBot","Google-Extended","Amazonbot","CCBot"]}
{"domain":"theage.com.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":52,"stanza_tokens":["*","Google-CloudVertexBot","Google-Extended","ChatGPT-User","GPTBot","OAI-SearchBot","OAISearch","anthropic-ai","claude-web","claudebot","facebookbot","meta-externalagent","meta-externalfetcher","applebot-extended","perplexitybot","cohere-ai","youbot","amazonbot","aliyunsecbot","audigentadbot","awariorssbot","awariosmartbot","blexbot","bytespider","ccbot","dataforseobot","diffbot","duckassistbot","echoboxbot","friendlycrawler","ia_archiver","imagesiftbot","mycentralaiscraperbot","newsnow","news-please","omgili","omgilibot","webzio-extended","peer39_crawler","peer39_crawler/1.0","quillbot.com","quora-bot","scrapy","seekrbot","seznamhomepagecrawler","taragroup intelligent bot","timpibot","turnitinbot","viennatinybot","jetslide","magpie-crawler","poseidon research crawler"]}
{"domain":"airtel.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","GPTBot","PerplexityBot","bingbot","BingPreview","bingGPT","Googlebot","Google-Extended","Bard","Gemini","ClaudeBot","Slurp"]}
{"domain":"stubhub.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"code42.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"knowyourmeme.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"immersivelabs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nj.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ameli.fr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pinduoduo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["Baiduspider-image"]}
{"domain":"pp.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"shopee.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"cuni.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"orange.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"cldin.eu","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"outlookindia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","PerplexityBot","Claude-Web","Bingbot","Meta-ExternalAgent","DeepSeekBot","Dalvik","*Dalvik*"]}
{"domain":"pyrus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["adsbot-google","*","Yandex","GoogleBot"]}
{"domain":"helsinki.fi","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"miamiherald.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"kzhi.tech","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"independent.ie","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["*","Googlebot-News","Amazonbot","anthropic-ai","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","Diffbot","FacebookBot","Google-Extended","GPTBot","magpie-crawler","omgili","omgilibot","PerplexityBot","meta-externalagent","meta-externalfetcher","Google-CloudVertexBot","Ahrefsbot","Archive.org_bot","Bravebot","Claude-Searchbot","Claude-User","DeepSeekBot","Meta-WebIndexer","MistralAI-Index","MistralAI-User","OAI-AdsBot","OAI-Searchbot","Perplexity-User","Youbot"]}
{"domain":"caranddriver.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AmazonAdBot","Google-InspectionTool","ChatGPT-User","GPTBot","OAI-SearchBot","*"]}
{"domain":"dartmouth.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Roverbot","*"]}
{"domain":"hiddenvpn.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"emirates.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"acehardware.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Googlebot","Googlebot-image","Storebot-Google","Google-Extended","*"]}
{"domain":"189.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"hereapi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"principal.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"3ds.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","GPTBot","ChatGPT-User","Claude-Web","ClaudeBot","PerplexityBot","Applebot-Extended","cohere-ai","Bytespider","CCBot","MistralAI-User","Google-Extended"]}
{"domain":"nodacdn.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"yandex-team.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"veloxzone.com.br","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"blackspider.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"wolt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"karte.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ECONNRESET"]}
{"domain":"kroger.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"18.in-addr.arpa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tik.porn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["archive.org_bot","ia_archiver","CCBot","*"]}
{"domain":"expressvpn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Googlebot","Yeti","Naverbot","ia_archiver","AhrefsBot"]}
{"domain":"nespresso.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"moma.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wikidata.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["MJ12bot","Mediapartners-Google*","IsraBot","Orthogaffe","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","fast","wget","grub-client","k2spider","NPBot","WebReaper","SemrushBot","*"]}
{"domain":"tiktokw.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"edgekey88.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"gr.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"jma.go.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"ms.now","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","anthropic-ai"],"open":[],"path_restricted":["Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":51,"stanza_tokens":["*","GPTBot","PerplexityBot","Amazonbot","anthropic-ai","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","HuggingFaceBot","ChatGPT-User","ClaudeBot","Claude-Web","DataForSeoBot","Diffbot","FacebookBot","Google-Extended","Google-NotebookLM","Google-Agent","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","Scrapy","TurnitinBot","cohere-ai","Applebot-Extended","Meta-ExternalAgent","meta-externalagent","OAI-SearchBot","Claude-User","polymarket-news-bot","KalshiBot","polymarket-bot","OpenClaw","gold-ai-event-bridge","TradingBot","trading-pipeline","kalshi-bot","BivariateKalshiPipeline","Falcon Trading Platform","StockMarketAnalyser","FinanceApp","polyclaw-ingester","IsoxyTrader","Neo Trading Bot","MerlinChartWizard","investment-dashboard"]}
{"domain":"napps-1.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"dayforcehcm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"liquidweb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","Google-Extended","PerplexityBot","Applebot-Extended","CCBot","bingbot","NerdyBot","duggmirror","Bytespider"]}
{"domain":"hatenablog.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dallasnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User","anthropic-ai"],"open":[],"path_restricted":["Claude-SearchBot"]}],"stanza_token_count":31,"stanza_tokens":["ai2bot-dolma","anthropic-ai","applebot-extended","bytespider","ccbot","claude-user","claude-web","claudebot","chatGPT-User","cohere-ai","diffbot","duckassistbot","Friendlycrawler","GPTBot","Grokbot","xAI-Grok","xAI","omgili","omgilibot","perplexity-user","perplexitybot","scrapy","timpibot","youbot","webzio-extended","kangaroo-bot","Spinn3r","TaraGroup Intelligent Bot","google-extended","AmazonAdBot","*"]}
{"domain":"myportfolio.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"aarnet.net.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"apple.com.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adlightning.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"manualslib.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":16,"stanza_tokens":["Mediapartners-Google","*","Baiduspider","AddThis.com robot tech.support@clearspring.com","AddThis.com","AhrefsBot","PetalBot","GPTBot","YandexBot","YandexRenderResourcesBot","YandexMetrika","ClaudeBot","Sogou web spider","meta-externalagent","Brightbot","CriteoBot"]}
{"domain":"b2clogin.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"libreoffice.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"gmu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"etherscan.io","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"cbp.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","PetalBot"]}
{"domain":"dynadot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["OAI-SearchBot","*"]}
{"domain":"pushd.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ucr.edu","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"counterpane.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","ECONNRESET"]}
{"domain":"widgetable.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"laposte.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mail.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":90,"stanza_tokens":["*","grub-client","grub","looksmart","WebZip","larbin","b2w/0.1","psbot","Python-urllib","NetMechanic","URL_Spider_Pro","CherryPicker","EmailCollector","EmailSiphon","WebBandit","EmailWolf","ExtractorPro","CopyRightCheck","Crescent","SiteSnagger","ProWebWalker","CheeseBot","LNSpiderguy","ia_archiver","ia_archiver/1.6","Teleport","TeleportPro","MIIxpc","Telesoft","Website Quester","moget/2.1","WebZip/4.0","WebStripper","WebSauger","WebCopier","NetAnts","Mister PiX","WebAuto","TheNomad","WWW-Collector-E","RMA","libWeb/clsHTTP","asterias","httplib","turingos","spanner","InfoNaviRobot","Harvest/1.5","Bullseye/1.0","Mozilla/4.0 (compatible; BullsEye; Windows 95)","Crescent Internet ToolPak HTTP OLE Control v.1.0","CherryPickerSE/1.0","CherryPickerElite/1.0","WebBandit/3.50","NICErsPRO","Microsoft URL Control - 5.01.4511","DittoSpyder","Foobot","WebmasterWorldForumBot","SpankBot","BotALot","lwp-trivial/1.34","lwp-trivial","BunnySlippers","Microsoft URL Control - 6.00.8169","URLy Warning","Wget/1.6","Wget/1.5.3","Wget","LinkWalker","cosmos","moget","hloader","humanlinks","LinkextractorPro","Offline Explorer","Mata Hari","LexiBot","Web Image Collector","The Intraformant","True_Robot/1.0","True_Robot","BlowFish/1.0","JennyBot","MIIxpc/4.2","BuiltBotTough","ProPowerBot/2.14","BackDoorBot/1.0","toCrawl/UrlDispatcher","AmazonAdBot"]}
{"domain":"ria.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","Baiduspider","GPTBot","Meta-ExternalAgent"]}
{"domain":"youversionapi.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"msftauthimages.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"meli.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"tripadvisor.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":25,"stanza_tokens":["Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Cohere-ai","GPTBot","Google-CloudVertexBot","Google-Extended","Img2dataset","PanguBot","PetalBot","SentiBot","YouBot","magpie-crawler","meta-externalagent","*","ChatGPT-User","ChatGPT-User/2.0","Gemini-Deep-Research","OAI-SearchBot","applebot","PerplexityBot","bingbot","Yandex"]}
{"domain":"aboutamazon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"interfax.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","YandexBot","GoogleBot","TelegramBot"]}
{"domain":"seek.com.au","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"uni-muenchen.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"livesportmedia.eu","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"byteeffecttos-g.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mass.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bbtec.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","UND_ERR_CONNECT_TIMEOUT"]}
{"domain":"cloudflare-gateway.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"rutracker.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["archive.org_bot","bingbot","DuckDuckBot","Googlebot","Googlebot-Mobile","ia_archiver","Yandex","Adsbot-Google","AhrefsBot","Googlebot-Image","Mail.Ru","Mediapartners-Google","msnbot","msnbot-media","Slurp","StackRambler","Yahoo-MMCrawler","*"]}
{"domain":"ixbt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot-News","gsa-crawler","Yandex"]}
{"domain":"webdav.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cloudrc.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"EAI_AGAIN","attempts":["EAI_AGAIN","EAI_AGAIN"]}
{"domain":"laravel-news.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"stripchatgirls.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","ClaudeBot","Amazonbot","facebookexternalhit"]}
{"domain":"ap.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ETIMEDOUT","TimeoutError"]}
{"domain":"nazwa.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"espressif.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tlivesource.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"exacttarget.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"gitv.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"telecomitalia.it","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"yadro.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"dns.kr","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vps-vids.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"bancointer.com.br","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"utah.gov","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"crazymaplestudios.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"wikiquote.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["MJ12bot","Mediapartners-Google*","IsraBot","Orthogaffe","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","fast","wget","grub-client","k2spider","NPBot","WebReaper","SemrushBot","*"]}
{"domain":"cadenaser.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["PetalBot","bedrock-agentcore-browser","Amazon-Bot","Amazonbot","AmazonProductDiscoveryBot","Amzn-SearchBot","Amzn-User","Arquivo-web-crawler","bnf.fr_bot","Bytespider","Claude-Extension","ClaudeBot","Claude-User","Claude-SearchBot","Comet","CloudFlare-AlwaysOnline","Cloudflare-Healthchecks","CloudflareBrowserRendering","Cloudflare-Radar","CCBot","DeepSeekBot","DuckAssistBot","archive.org_bot","ia_archiver","MistralAI-User","Moreover","Adbeat","Argus","Baiduspider","Baiduspider-image","Bitdefender","Blogtrottr","BloombergBot","BoardReader","checkmy.ws","Clickagy","CookieBot","cXensebot","DataObserver","DomainStatsBot","Embedly","Feedspotbot","FlipboardProxy","Getintent","Google-Apps-Script","GoogleDrive","Grapevine6","Gumgum","HaloBot","HubSpot Crawler","IbouBot","Inout","LinerBot","MJ12bot","Manus","Meltwater","Metajob","Muckrack","NetSeer","Netvibes","Onyphe","OpenClaw","Opoint","OutbrainBot","PlumAnalytics","ProSitemaps","PerplexityBot","Perplexity-User","proximic","PubMatic","Quantcastbot","SearchConcepts","Semantic-Visions","SentiBot","Sindup","Sirdata","SmartologyBot","Superfeedr","surlybot","Synaptive","TinEye","Trendiction","TurnitinBot","Ubermetrics","Vebidoo","Webcorp","Weborama","WebPageTest","YandexAccessibilityBot","YandexDirect","YandexDirectDyn","YaDirectFetcher","YandexFavicons","YandexMetrika","YandexMobileBot","YandexNews","YandexRCA","YandexRenderResourcesBot","YandexUserproxy","*"]}
{"domain":"windstream.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":44,"stanza_tokens":["*","ADmantX","AlphaBot","anthropic-ai","AwarioRssBot","AwarioSmartBot","BLEXBot","Buzzbot","Bytespider","CCBot","ChatGPT-User","claritybot","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","huggingface","ImagesiftBot","img2dataset","magpie-crawler","Meltwater","Neevabot","news-please","NewsNow","Nutch","omgili","omgilibot","panscient.com","Perplexity-ai","PerplexityBot","PetalBot","PiplBot","scoop.it","Scrapy","Seekr","SentiBot","SeznamBot","TurnitinBot","YouBot","ZumBot"]}
{"domain":"rj.link","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"quicksetcloud.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"depositphotos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kremlin.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"digitalcaramel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"temp-mail.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"apartments.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"weixinbridge.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rs-online.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"abcp.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thinkingdata.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hsadspixel.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tobsnssdk.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hubpages.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","Googlebot","Mediapartners-Google","Covario-IDS","AhrefsBot","Owlin","WBSearchBot","PiplBot","XoviBot","magpie-crawler","Yandex"]}
{"domain":"roche.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fastcgi.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","UND_ERR_CONNECT_TIMEOUT","TimeoutError"]}
{"domain":"bleepingcomputer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":26,"stanza_tokens":["Slurp","Applebot-Extended","SeekportBot","AhrefsBot","StractBot","AwarioRssBot","AwarioSmartBot","ias_crawler","MJ12bot","Cliqzbot","DotBot/1.1","MauiBot","NTENTbot","CCBot","sentibot","trendictionbot","yacybot","iSec_Bot","BLEXBot","AhrefsBot/6.1","Qwantify/Bleriot/1.1","DemandbasePublisherAnalyzer/0.1","weborama-fetcher","ClaudeBot","*","AmazonAdBot"]}
{"domain":"scitechdaily.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"service-public.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"secondlife.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"qualys.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"startribune.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":[429,429,429]}
{"domain":"youdao.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"jpmorgan.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fu-berlin.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"forrester.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","GPTBot","ClaudeBot","Claude-User","Claude-SearchBot","Google-Extended","CCBot","FacebookBot","Omgilibot"]}
{"domain":"httpbin.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"conn3ct.space","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"7-zip.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"fastpanel.direct","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"kwaipay.app","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"2kcoretech.online","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"xiaoyi.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"l-agent.me","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"acorns.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aliyuncsslb.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"aa.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bnf.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"3m.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"mcafee.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"adtidy.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"smooch.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"ggblueshark.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"sleepycat.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"connman.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"rsasecurity.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"gscdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"globalconnetct.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"cybertrust.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"dns.id","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"skynet.be","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"dailyinnovation.biz","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":[503,503,"TimeoutError"]}
{"domain":"benzinga.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","magpie-crawler","scoop.it","Scrapy"]}
{"domain":"webtoons.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot-Image","Scrapy"]}
{"domain":"chinanews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"simply.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"pochta.ru","fetch_ok":false,"usable":false,"reason":"other","status":417,"error":null,"attempts":["TimeoutError",417]}
{"domain":"lapresse.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":52,"stanza_tokens":["*","AhrefsBot","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","archive.org_bot","Archive-It","BaiduSpider","BLP_bbot","Brandwatch","Bytespider","CCBot","ChatGPT-User","Cision","Claude","ClaudeBot","Claude-SearchBot","Claude-Web","DataForSeoBot","Diffbot","dotbot/1.0","FacebookBot","Gigabot","Gnowit","GnowitNewsbot","Google-Extended","GPTBot","Heritrix","ia_archiver","linkfluence","magpie-crawler","mediatoolkit.com","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","Muckrack","OAI-SearchBot","openai","OTMedia","Perplexity","PerplexityBot","scoopit","scpitspi-rs","semantic-visions.com","SemrushBot","SentiBot","trendiction","Turnitin","webzio","webzio-extended","YouBot"]}
{"domain":"activision.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"oilprice.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":24,"stanza_tokens":["*","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Claude-Web","ClaudeWeb","cohere-ai","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","ImagesiftBot","img2dataset","Meta-ExternalAgent","omgili","omgilibot","Scrapy","Timpibot","VelenPublicWebCrawler","YouBot"]}
{"domain":"html5up.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mexc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"basecamp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"breitbart.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Mediapartners-Google","Adsbot-Google","Googlebot-Image","Googlebot-Mobile","ClaudeBot"]}
{"domain":"rocketreach.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lazada.co.th","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nato.int","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"csmonitor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":36,"stanza_tokens":["Mediapartners-Google*","NetShelter ContentScan","NetShelter","^NetShelter","sentibot","anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ClaudeBot","Claude-Web","cohere-ai","Diffbot","FacebookBot","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","GPTBot","ImagesiftBot","magpie-crawler","Meta-ExternalAgent","meta-externalagent","MyCentralAIScraperBot","NewsNow","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Poseidon Research Crawler","quillbot.com","Scrapy","Timpibot","*"]}
{"domain":"experian.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["PetalBot","*"]}
{"domain":"copper6.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"manchester.ac.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kwai-pay.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"nuro.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mediation.goog","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"ggpolarbear.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"yummyani.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"netnod.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cdnwidget.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"streamable.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":10,"stanza_tokens":["*","msnbot","Bingbot","AdIdxBot","BingPreview","Yandex","GPTBot","ChatGPT-User","CCBot","SemrushBot"]}
{"domain":"puma.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"dnspod.cn","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"mediaset.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nanit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","adsbot-google"]}
{"domain":"eltiempo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":61,"stanza_tokens":["*","Googlebot","Googlebot-News","Googlebot-Image","Googlebot-Video","Googlebot-Mobile","AdsBot-Google","AdsBot-Google-Mobile","Mediapartners-Google","Google-Extended","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","UbiCrawler","DOC","Zao","Slurp","newsproxy.app","PerplexityBot","TurnitinBot","magpie-crawler","omgili","omgilibot","DataForSeoBot","Diffbot","news-please","AwarioRssBot","AwarioSmartBot","Bytespider","archive.org_bot","archive-it.org","Maxthon","CNCDialer","CCBot","ChatGPT Agent","ClaudeBot","Claude-Web","GPTBot","anthropic-ai","ChatGPT-User","OAI-SearchBot","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Meta-WebIndexer"]}
{"domain":"akasripcn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cognitivlabs.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"jcpenney.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ijg.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"nhtsa.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"amazon.ie","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":100,"stanza_tokens":["*","EtaoSpider","GPTBot","CCBot","PerplexityBot","Google-Extended","GoogleAgent-Mariner","GoogleAgent-Shopping","ClaudeBot","meta-externalagent","Bytespider","Scrapy","PetalBot","Devin","omgili","AI2Bot","Gemini-Deep-Research","PanguBot","MistralAI-User","Diffbot","Sidetrade indexer bot","Google-NotebookLM","DuckAssistBot","Copilot","Ai2Bot-Dolma","Google-CloudVertexBot","meta-externalfetcher","iaskspider","wpbot","aiHitBot","cohere-ai","FriendlyCrawler","img2dataset","VelenPublicWebCrawler","YouBot","Brightbot","Claude-User","Claude-SearchBot","Perplexity-User","ISSCyberRiskCrawler","CopilotNative","CopilotSapphire","imgproxy","SemrushBot-SWA","ChatGPT-User","OAI-SearchBot","Timpibot","Thinkbot","AddSearchBot","AI2Bot-DeepResearchEval","Andibot","Anomura","atlassian-bot","BuddyBot","Channel3Bot","ChatGLM-Spider","Claude-Web","Cloudflare-AutoRAG","CloudflareBrowserRenderingCrawler","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Echobot Bot","EchoboxBot","factset_spyderbot","Google-Firebase","GoogleOther","Grok-DeepSearch","GrokBot","iAskBot","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","Linguee Bot","LinkupBot","Manus-User","meta-webindexer","netEstate Imprint Crawler","Panscient","panscient.com","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","ShapBot","TavilyBot","TerraCotta","TwinAgent","webzio-extended","WRTNBot","xAI-Grok","YaK","ZanistaBot"]}
{"domain":"nifty.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thinkwithgoogle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bhphotovideo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AdsBot-Google-Mobile"]}
{"domain":"uservoice.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["TermlyBot","*"]}
{"domain":"cointelegraph.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uconn.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ussp.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":22,"stanza_tokens":["*","YandexAccessibilityBot","YandexCalendar","YandexDialogs","YandexDirect","YandexDirectDyn","YandexFavicons","YaDirectFetcher","YandexMobileBot","YandexMarket","YandexMetrika","YandexMobileScreenShotBot","YandexOntoDBAPI","YandexPartner","YandexRCA","YandexRenderResourcesBot","YandexSearchShop","YandexTracker","YandexUserproxy","YandexVideoParser","YandexScreenshotBot","YandexAdditionalBot"]}
{"domain":"activemetering.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bilicdn1.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gdflpr.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.jo","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"doxygen.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"wallpapers.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"myrostelecom.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"EAI_AGAIN","attempts":["EAI_AGAIN","EAI_AGAIN"]}
{"domain":"aclu.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"biblehub.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ropro.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"pearson.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sputniknews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"phpmyadmin.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tinder.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"webshare.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"icq.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","YandexDirect"]}
{"domain":"csic.es","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"xtracloud.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"detailroi.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"tdnsvod1.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gjhyss.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"domainnamedns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"invisioncic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"larksuite.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ortb.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"serverwatch.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uic.edu","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","UNABLE_TO_VERIFY_LEAF_SIGNATURE","UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"saasprotection.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_PACKET_LENGTH_TOO_LONG","attempts":["ERR_SSL_PACKET_LENGTH_TOO_LONG","TimeoutError"]}
{"domain":"yougov.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Bingbot","*"]}
{"domain":"msauthimages.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"claro.net.br","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"tekioncloud.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"channelcom.tech","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"t.ly","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"neocities.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ons.gov.uk","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"manutouch.com.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","UND_ERR_SOCKET"]}
{"domain":"9mod.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"1cfresh.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Riddler","*"]}
{"domain":"krone.at","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rnp.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mycdn.me","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"ipwho.is","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"michigan.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"zebra.com","fetch_ok":false,"usable":false,"reason":"other","status":406,"error":null,"attempts":[406,406]}
{"domain":"dual-s-dc-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kameleoon.eu","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kotaku.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["*","AmazonAdBot","AI2Bot","Bytespider","CCBot","Cohere-ai","Diffbot","FirecrawlAgent","ImagesiftBot","Kangaroo Bot","Meta-ExternalAgent","Omgilibot","PanguBot","Timpibot","Webzio-Extended","HTTrack","Nutch","Scrapy"]}
{"domain":"blick.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Flamingo_SearchEngine","ExaBot","dotbot","genieo","crystalsemantics","grapeshot","ClaudeBot","AmazonAdBot"]}
{"domain":"makeuseof.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","\"008\"","anthropic-ai","Applebot","Applebot-extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","GPTBot","HTTrack","Nutch","Offline Explorer","omgili","Scrapy","YouBot"]}
{"domain":"bookshop.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":19,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","PerplexityBot","Claude-SearchBot","Claude-User","YouBot","DuckAssistBot","GPTBot","Google-Extended","ClaudeBot","Amazonbot","CCBot","Bytespider","Meta-ExternalAgent","Applebot-Extended","cohere-ai","Diffbot","Baiduspider"]}
{"domain":"uio.no","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["SolrVortexConnector","*"]}
{"domain":"a-mx.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"aistrm.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ketchcdn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":53,"stanza_tokens":["*","bingbot","Baiduspider","YandexBot","Slurp","GPTBot","ChatGPT-User","ChatGPT-User/2.0","OAI-SearchBot","CCBot","anthropic-ai","ClaudeBot","claude-web","PerplexityBot","Perplexity-User","MistralAI-User","Google-Extended","Amazonbot","Applebot-Extended","cohere-ai","Diffbot","omgili","YouBot","Bytespider","AhrefsBot","SemrushBot","MJ12bot","DotBot","PetalBot","BLEXBot","DataForSeoBot","HTTrack","wget","curl","WebCopier","WebZIP","Teleport","WebReaper","WebSauger","WebStripper","WebSucker","WebWhacker","emailcollector","emailsiphon","emailwolf","Nmap","Nikto","sqlmap","masscan","ZmEu","Acunetix","Nessus","OpenVAS"]}
{"domain":"kijiji.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"rgnameserver.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"frameo.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"goto.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wassenaar.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"dotvndns.vn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"gscdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hugedomainsdns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mercadolibre.cl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User"],"open":[],"path_restricted":["Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":13,"stanza_tokens":["Amazonbot","PerplexityBot","Perplexity-User","ClaudeBot","Claude-User","GPTBot","ChatGPT-User","FacebookExternalHit","FacebookBot","Twitterbot","LinkedInBot","Bingbot","*"]}
{"domain":"spectrum.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ebay.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":15,"stanza_tokens":["*","bingbot","AdsBot-Google","AdIdxBot","Twitterbot","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","PerplexityBot","anthropic-ai","AmazonBot","meta-externalagent","GPTBot","Applebot-Extended"]}
{"domain":"frontiernet.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"usaa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Semrushbot-SA","Semrushbot-SI","PageFreezer","*"]}
{"domain":"techrepublic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","magpie-crawler"]}
{"domain":"cbr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"freebit.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"smule.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"gofile.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tconnect.services","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"DEPTH_ZERO_SELF_SIGNED_CERT","attempts":["DEPTH_ZERO_SELF_SIGNED_CERT","DEPTH_ZERO_SELF_SIGNED_CERT"]}
{"domain":"controld.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"google.ro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"overleaf.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zoneedit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"dfn.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"infobip.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":24,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","GPTBot","anthropic-ai","ClaudeBot","Claude-Web","CCBot","Bytespider","Meta-ExternalAgent","Meta-ExternalFetcher","cohere-ai","cohere-training-data-crawler","Google-Extended","AzureAI-SearchBot","bedrockbot","Amazonbot","Diffbot","img2dataset","Applebot-Extended"]}
{"domain":"emarketer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Google-Extended","*"]}
{"domain":"uky.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"illusionblack.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"upi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":9,"stanza_tokens":["*","AmazonAdBot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"zotero.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"chinanews.com.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"qodeinteractive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ip.sb","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"telogis.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"swarovski.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","Google-NotebookLM","Google-Extended","bingbot","BingPreview","MicrosoftPreview","ClaudeBot","Claude-SearchBot","PerplexityBot","Perplexity-User","CCBot","CazoodleBot","MJ12bot","dotbot/1.0","Gigabot"]}
{"domain":"zyxel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"webengage.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mercadolibre.com.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User"],"open":[],"path_restricted":["Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":13,"stanza_tokens":["Amazonbot","PerplexityBot","Perplexity-User","ClaudeBot","Claude-User","GPTBot","ChatGPT-User","FacebookExternalHit","FacebookBot","Twitterbot","LinkedInBot","Bingbot","*"]}
{"domain":"tv-asahi.co.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"vedomosti.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":33,"stanza_tokens":["Yandex","Googlebot","*","SemrushBot","BLEXBot","AhrefsBot","MJ12bot","OAI-SearchBot","ChatGPT-User","ChatGPT-User/2.0","GPTBot","anthropic-ai","ClaudeBot","claude-web","PerplexityBot","Perplexity-User","Google-Extended","BingBot","Amazonbot","Applebot","Applebot-Extended","FacebookBot","meta-externalagent","LinkedInBot","DuckAssistBot","Bytespider","cohere-ai","AI2Bot","CCBot","Diffbot","omgili","TimpiBot","YouBot"]}
{"domain":"five64dmv.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"emory.edu","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"imperial.ac.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["slurp","*"]}
{"domain":"queniuuf.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"devicetask.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rogers.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.com.kh","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"getcomposer.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"byteplusvod.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"amplience.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ora.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":65,"stanza_tokens":["*","Baiduspider","PetalBot","yisouspider","008","ChatGPT-User","ChatGPT-User/2.0","PerplexityBot","Perplexity-User","DuckAssistBot","Applebot","bingbot","Googlebot","ClaudeBot","claude-web","Claude-SearchBot","YouBot","Applebot-Extended","MistralAI-User","meta-webindexer","GoogleAgent-Mariner","DotBot","archive.org_bot","Pinterestbot","TelegramBot","kakaotalk-scrap","getstream.io/opengraph-bot","RamblerMail","Brave","OAI-SearchBot","GPTBot","anthropic-ai","Google-Extended","cohere-ai","CCBot","AI2Bot","Amazonbot","Amazonbot-Video","Bytespider","meta-externalagent","Diffbot","omgili","TimpiBot","SeznamBot","Exabot","YandexBot","Sogou","360Spider","YisouSpider","Baiduspider-Render/2.0","MJ12bot","DataForSeoBot","AliyunSecBot","ArchiveTeam","ArchiveTeam ArchiveBot","AwarioBot","ZoominfoBot","ali-implementer","Blueno","BIGO-baiguoyuan","WorksOgCrawler","OnPageBot","DoCoMo","SAMSUNG-SGH-E250","TA-Googlebot"]}
{"domain":"atera.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"gslbjpmchase.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"epfl.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"deccanherald.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":17,"stanza_tokens":["Mediapartners-Google","SeekportBot","Yandex","GPTBot","GPTCrawler","ChatGPT-User","OAI-SearchBot","ClaudeBot","anthropic-ai","Google-Extended","CCBot","PerplexityBot","Perplexity-User","Bytespider","Bytespider-webpage","Amazonbot","*"]}
{"domain":"sooplive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"buffalo.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Twitterbot","facebot","facebookexternalhit","bingbot","*"]}
{"domain":"gta5-mods.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"chinatimes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":42,"stanza_tokens":["Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","Meta-ExternalAgent","Meta-ExternalFetcher","FacebookBot","facebookexternalhit","*","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","Diffbot","DuckAssistBot","FriendlyCrawler","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","OAI-SearchBot","omgili","omgilibot","PerplexityBot","PetalBot","Scrapy","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot"]}
{"domain":"tinypic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pscp.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Twitterbot"]}
{"domain":"michelin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"apple.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kinoafisha.info","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["Yandex","GoogleBot","*","msnbot","bingbot","dotbot","SemrushBot","weborama-fetcher","AhrefsBot","DataForSeoBot","SeekportBot","serpstatbot","BLEXBot","ImagesiftBot","ClaudeBot"]}
{"domain":"pdf24.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"marzaent.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mosyle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fr.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":248,"stanza_tokens":["*","Amazonbot","AlibabaBot","Ai2Bot-Dolma","Applebot-Extended","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","Claude-Web","CloudVertexBot","cohere-training-data-crawler","Cotoyogi","Datenbank Crawler","Diffbot","FacebookBot","Google-Extended","GoogleOther","GPTBot","ICC-Crawler","imageSpider","Kangaroo Bot","laion-huggingface-processor","LCC","meta-externalagent","netEstate Imprint Crawler","omgili","PanguBot","PerplexityBot","SBIntuitionsBot","Spider","Timpibot","VelenPublicWebCrawler","webzio-extended","YouBot","008","Dataprovider.com","dcrawl","HelloworkJobPostingBot","HTTrack","HTTrack 3.0","KrawlerBot","MetaInspector","MetaJobBot","newspaper","Nutch","Offline Explorer","OpenindexSpider","Potions","Scrapy","ServerHunterSpider","StatsDroneBot","CloudflareBrowserRenderingCrawler","aa","AhrefsBot","Attracta","Barkrowler","BrightEdge Crawler","BLEXBot","CaliberBot","Caliperbot","ClarityBot","Clearscopebot","Cloudtrellis","cludo.com","Cocolyzebot","cognitiveSEO Crawler","contentking","Convermax","Cxense","DataForSeoBot","DataForSEO Bot","DomainStatsBot","DotBot","Dragonbot","Huckabot","Huckabuy Bot","hypestat","iaskspider","img2dataset","James BOT","LinkCheck by Siteimprove","LinkChecker Bot","linkchecker.pro","linkdexbot","LinksIndexerBot","MarketGoo","MBCrawler","MegaIndex.ru","MJ12bot","Moz dotbot","Moz rogerbot","Nitro-","NitroBot","online-webceo-bot","Prerender","Pro Sitemaps","Readable","RevvimGort","rogerbot-crawler"]}
{"domain":"icons8.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","GoogleBot","Yandex","ClaudeBot","Bytespider","CCBot","FacebookBot","GPTBot"]}
{"domain":"trtcube-license.cn","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":["ENOTFOUND",400]}
{"domain":"share-dns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"dns.ne.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kgns3.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"lyft.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AdsBot-Google","Googlebot-Image"]}
{"domain":"abc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":6,"stanza_tokens":["*","MJ12bot","Twitterbot","GPTBot","Google-Extended","trendkite-akashic-crawler"]}
{"domain":"comicbook.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wsukwai.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"swiftycdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kayak.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":21,"stanza_tokens":["*","Googlebot","Facebot","Twitterbot","ChatGPT-User","AdsBot-Google","AdsBot-Google-Mobile","Google-HotelAdsVerifier","Google-HotelAdsVerifier/2.0","Google-Travel-Flights","Mediapartners-Google","AdIdxBot","BUbiNG","proximic","A6-Indexer/1.0","ADmantX Platform Semantic Analyzer","ContextAd Bot","berlin-fu-cow","ias-ir","ias-sg","AhrefsBot"]}
{"domain":"ns.se","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"monster.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE",403]}
{"domain":"lexisnexis.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"myharmony.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bricklink.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":4,"stanza_tokens":["bingbot","Baiduspider","GPTBot","*"]}
{"domain":"adguard-dns.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"cdnhwctnm107.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.com.np","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"lufthansa.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pagesjaunes.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","MJ12bot","AhrefsBot","SemrushBot","dotbot","CCBot","Amazonbot","CriteoBot/0.1"]}
{"domain":"canon.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"china.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"prom.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["YadirectBot","YandexDirect","*"]}
{"domain":"bitrix24.tech","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"kdocs.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"siol.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adtdp.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"kustomerapp.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"streamabc.link","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"barracuda.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":28,"stanza_tokens":["*","DotBot","dotbot","Sogou Spider","Exabot","YoudaoBot","MegaIndex.ru","megaindex.com","SeznamBot","Cliqzbot","Gigabot","spbot","Ezooms","ScoutJet","BLEXBot","sistrix","SISTRIX Crawler","SISTRIX","SEOkicks","SEOkicks-Robot","SearchmetricsBot","SurveyBot","CCBot","DomainStatsBot","Sosospider","Teleport","TeleportPro","HTTrack"]}
{"domain":"globalvisionad.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"fstr.rocks","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rainbird.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"larazon.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":105,"stanza_tokens":["*","Googlebot","Googlebot-News","Googlebot-Image","Googlebot-Video","AmazonAdBot","Baiduspider","Yandex","008","UnisterBot","JikeSpider","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","larbin","libwww","ZyBORG","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Download Ninja","Maxthon","CNCDialer","wget","grub-client","k2spider","NPBot","WebReaper","TurnitinBot","MJ12bot","magpie-crawler","Flamingo_SearchEngine","Exabot","Moreoverbot","CyberAlert","Newscan","Spinn3r","LexxeBot/1.0","NextGenSearchBot","Sosospider","rogerbot","SiteBot/0.1","SiteBot","CrystalSemanticsBot","NetSeer crawler","trovitBot","DotBot","Ezooms","discobot","Jyxobot","sogou","heritrix","NerdByNature.Bot","psbot","WBSearchBot","http://addthis.com/","discoverybot","bl.uk_lddc_bot","IstellaBot","seokicks","Bender","wotbox","Yasni","netEstate NE Crawler","Pixray-Seeker","integromedb","BLEXBot","BDCbot","WeSEE:Search","admantx","spbot","BUbiNG","anthropic-ai","AI2Bot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","cohere-training-data-crawler","Cotoyogi","Datenbank Crawler","Diffbot","FacebookBot","Factset_spyderbot","GPTBot","ICC-Crawler","Kangaroo Bot","Meta-ExternalAgent","netEstate Imprint Crawler","omgili"]}
{"domain":"test.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"rtbwise.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ipecho.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"thespruce.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"hwg.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"serveriai.lt","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"cdn-fc.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"idnscloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"laztys.tech","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gla.ac.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cloudflareportal.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"mullvad.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"toolmatrix.plus","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"game8.co","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"247sports.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"104.com.tw","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"aip.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["bingbot"]}
{"domain":"media6degrees.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gamebanana.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"reference.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"vodafone.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","SlySearch","SmartDownload","SpankBot","spanner","SuperBot","SupperHTTP","Surfbot","ia_archiver"]}
{"domain":"zalan.do","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"architecturaldigest.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","FacebookExternalHit","Twitterbot","Pinterestbot","LinkedInBot","TikTokSpider","Storebot-Google","Google-InspectionTool","AmazonAdBot","Google-Extended","Google-CloudVertexBot","GoogleOther","Applebot-Extended","Amazonbot","meta-webindexer","meta-externalagent","meta-externalfetcher","ClaudeBot","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","cohere-training-data-crawler","cohere-ai","CCBot","PanguBot","PetalBot","Bytespider","Diffbot","DuckAssistBot","MistralAI-User","Timpibot","Webzio","Webzio-Extended","archive.org_bot","ia_archiver","ia_archiver-web.archive.org","heritrix"]}
{"domain":"rylmtch.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"vizio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"allocine.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["AmazonAdBot","*","seekbot","Exabot","Slurp","Cliqzbot","Arquivo-web-crawler","AI2Bot","ChatGLM","ChatGLM-spider","cohere-training-data-crawler","Diffbot","OmigiliBot","PetalBot","YouBot"]}
{"domain":"mobileiron.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"muni.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["facebookexternalhit","*"]}
{"domain":"blogspot.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"codecademy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eyeo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pdflib.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"blackrock.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"azurerms.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","UND_ERR_CONNECT_TIMEOUT"]}
{"domain":"geo.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"wto.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"expressen.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":32,"stanza_tokens":["CCBot","ChatGPT-User","GPTBot","Google-Extended","Google-CloudVertexBot","Omgilibot","Omgili","FacebookBot","ClaudeBot","Claude-SearchBot","Claude-User","Diffbot","DuckAssistBot","PerplexityBot","Perplexity-User","cohere-ai","cohere-training-data-crawler","Meta-ExternalAgent","Meta-ExternalFetcher","Meta-WebIndexer","Timpibot","Webzio-Extended","YouBot","Amazonbot","Bytespider","anthropic-ai","OAI-SearchBot","VelenPublicWebCrawler","Gemini-Deep-Research","ICC-Crawler","MistralAI-User","*"]}
{"domain":"sweetwater.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","bingbot","msnbot","Pinterestbot","Googlebot"]}
{"domain":"tripadvisor.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":25,"stanza_tokens":["Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Cohere-ai","GPTBot","Google-CloudVertexBot","Google-Extended","Img2dataset","PanguBot","PetalBot","SentiBot","YouBot","magpie-crawler","meta-externalagent","*","ChatGPT-User","ChatGPT-User/2.0","Gemini-Deep-Research","OAI-SearchBot","applebot","PerplexityBot","bingbot","Yandex"]}
{"domain":"doctolib.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["ia_archiver","Meta-ExternalAgent","*"]}
{"domain":"ohio.gov","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"zepto.co.in","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rdcnw.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"filmaffinity.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"theepochtimes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot"],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":["ClaudeBot"],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":16,"stanza_tokens":["LinkedInBot","Scope3/2.0 (scope3.com)","*","Twitterbot","facebookexternalhit","OAI-SearchBot","PerplexityBot","ClaudeBot","Google-Extended","GPTBot","anthropic-ai","omgilibot","omgili","Mediatoolkitbot","PolecatBot","DotBot"]}
{"domain":"booth.pm","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"queue-it.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot, bingbot, BingPreview, Yandex, DuckDuckBot, AhrefsBot","*"]}
{"domain":"bentley.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"teachable.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","ClaudeBot","PerplexityBot"]}
{"domain":"g2a.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"deepl-cdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ENOTFOUND"]}
{"domain":"royalmail.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"we.tl","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"elcorteingles.es","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"makerworld.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"warwick.ac.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["SleepBot","*","Rogerbot"]}
{"domain":"flysleep.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"onpe.gob.pe","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",403]}
{"domain":"capcutvod.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sunrise.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Sogou","YoudaoBot"]}
{"domain":"pingone.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cuhk.edu.hk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kontur.host","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"second-ns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"byteintlapi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"9to5mac.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"sundaysky.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dndbeyond.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Mediapartners-Google*","TagInspector"]}
{"domain":"gotomeeting.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"filesusr.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"vodafone.pt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","cludo"]}
{"domain":"stake.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"atptour.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"myworkday.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"mheducation.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Google-Extended"]}
{"domain":"docusign.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tailwindcss.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"demonii.com","fetch_ok":false,"usable":false,"reason":"other","status":500,"error":null,"attempts":[500,500,500]}
{"domain":"ponyexpress.tech","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"goodrx.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","TagInspector"]}
{"domain":"freetype.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"justgiving.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["MJ12bot","Googlebot-News","Googlebot","Bingbot","Slurp","DuckDuckBot","Baiduspider","YandexBot","*"]}
{"domain":"churchofjesuschrist.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kugou.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"lilly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"missouri.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"barracudanetworks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":28,"stanza_tokens":["*","DotBot","dotbot","Sogou Spider","Exabot","YoudaoBot","MegaIndex.ru","megaindex.com","SeznamBot","Cliqzbot","Gigabot","spbot","Ezooms","ScoutJet","BLEXBot","sistrix","SISTRIX Crawler","SISTRIX","SEOkicks","SEOkicks-Robot","SearchmetricsBot","SurveyBot","CCBot","DomainStatsBot","Sosospider","Teleport","TeleportPro","HTTrack"]}
{"domain":"nj.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","ia_archiver","archive.org_bot","heritrix","My-heritrix-crawler","Timpibot","omgili","Bytespider","DuckAssistBot","Applebot-Extended","Google-Extended","Google-CloudVertexBot","GoogleOther","GPTBot","CCBot","Amazonbot","FacebookBot","cohere-ai","cohere-training-data-crawler","Omgilibot","ClaudeBot","Diffbot","PanguBot","MistralAI-User","webz.io","DeepSeekBot","AmazonAdBot","Amzn-SearchBot","Amzn-User","Claude-SearchBot","Claude-User","PetalBot","Meta-ExternalFetcher","Meta-WebIndexer","OAI-SearchBot","ChatGPT-User","PerplexityBot","Perplexity-User"]}
{"domain":"anteldata.com.uy","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"web.nhk","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":15,"stanza_tokens":["GPTBot","ChatGPT-User","ClaudeBot","Google-Extended","Applebot-Extended","anthropic-ai","cohere-ai","CCBot","ICC-Crawler","Bytespider","OAI-SearchBot","PerplexityBot","Meta-ExternalAgent","Perplexity-ai","Perplexity-User"]}
{"domain":"dssedge.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rapidedge.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vfsglobal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"olx.in","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"gmru.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bleacherreport.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":6,"stanza_tokens":["*","GPTBot","Google-Extended","CCBot","FacebookBot","anthropic-ai"]}
{"domain":"avs.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hsprotect.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"tractorsupply.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"qiniudns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"tiscali.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mx-ns.mx","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"minecraftservices.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dyndns.tv","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"google.ie","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"zmaticoo.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ask.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"free-reels.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"rainews.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":28,"stanza_tokens":["*","Twitterbot","FacebookBot","ChatGPT-User","Google-Extended","GPTBot","PerplexityBot","Amazonbot","anthropic-ai","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ClaudeBot","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","Scrapy","TurnitinBot","Botify"]}
{"domain":"nic.do","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"revolut.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sleepfoundation.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"olx.ro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dba.dk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jobstreet.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"shrm.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Pingdom.com_bot","Blackbox Exporter"]}
{"domain":"boeing.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uva.nl","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"thomann.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":7,"stanza_tokens":["*","Storebot-Google","Mediapartners-Google","AdsBot-Google","AdsBot-Google-Mobile-Apps","DataForSeoBot","GPTBot"]}
{"domain":"coppel.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"revolve.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"dropz-k.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nbcsports.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AmazonAdBot","*"]}
{"domain":"bingapis.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["msnbot-media","Twitterbot","*"]}
{"domain":"ticketmaster.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"oclc.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","two"]}
{"domain":"serasa.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"metal-archives.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"geocaching.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["dotbot","Slurp","twiceler","ScoutJet","Fasterfox","*"]}
{"domain":"catsonictp.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"thebalancemoney.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"fubo.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"kubernetes.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"githubassets.com","fetch_ok":false,"usable":false,"reason":"other","status":500,"error":null,"attempts":["ENOTFOUND",500,500]}
{"domain":"feverup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":106,"stanza_tokens":["*","008","AddThis.com","admantx","AhrefsBot","BDCbot","Bender","BIXOCRAWLER","bl.uk_lddc_bot","BLEXBot","BUbiNG","Cliqzbot","CNCDialer","crawler4j","CrystalSemanticsBot","CyberAlert","DigExt","discobot","discoverybot","dloader","DOC","dotbot","Download Ninja","DTS Agent","Exabot","Ezooms","FairShare","Fetch","Flamingo_SearchEngine","Genieo","Gigabot","grub-client","Heritrix","heritrix/3.3.0","HTTrack","ia_archiver","integromedb","IstellaBot","JikeSpider","Jyxobot","k2spider","Kimengi","Kimengi/nineconnections.com","larbin","LexxeBot/1.0","libwww","linko","Livelapbot","magpie-crawler","Maxthon","MetaURI","Microsoft.URL.Control","MJ12bot","Moreover","Moreoverbot","MSIECrawler","nabot","NerdByNature.Bot","netEstate NE Crawler","NetSeer crawler","Newscan","NextGenSearchBot","NPBot","nutch","Offline Explorer","omgilibot","Orthogaffe","PiplBot","Pixray-Seeker","proximic","psbot","QuerySeekerSpider","rogerbot","seokicks","SEOkicks-Robot","SiteBot","SiteBot/0.1","sitecheck.internetseer.com","SiteSnagger","sogou","Sosospider","spbot","Spinn3r","Teleport","TeleportPro","trendictionbot","trovitBot","TurnitinBot","UbiCrawler","uMBot-LN","UnisterBot","UniversalFeedParser","WBSearchBot","WebCopier","WebReaper","WebStripper","WebZIP","WeSEE:Search","wget","Wotbot"]}
{"domain":"splunk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ice.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rozetka.com.ua","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.al","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"join.chat","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"apache-ssl.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"menlosecurity.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"emerald.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["008","*","ChatGPT-User","OAI-SearchBot","Perplexity-User","PerplexityBotClaude-SearchBot","Claude-User"]}
{"domain":"ricta.org.rw","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","ETIMEDOUT"]}
{"domain":"sudouest.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":["OAI-SearchBot"],"path_restricted":[]}],"stanza_token_count":130,"stanza_tokens":["Adsbot-Google","AdsBot-Google-Mobile","APIs-Google","Applebot","Bingbot","bnf.fr_bot","exabot","Googlebot","Googlebot-Image","Googlebot-Mobile","Googlebot-News","Googlebot-Video","Googlebot_Nauxeo","ia_archiver","Mediapartners-Google","Proximic","Slurp","Storebot-Google","Twitterbot","upday","sistrix","BeOpBot","weborama-fetcher","Flipboard","FlipboardProxy","AmazonAdBot","Meltawer","Digimind","Knowings","Sindup","Cision","Talkwater","TurnitinBot","ConveraCrawler","Jetbot","NewsNow","kbcrawl","AmiSoftware","Newzbin","Ask n read","Qwam content intelligence","Zite","Youmag","Synthesio","trendybuzz","spotter","scoop.it","linkfluence","5emeRue","Augure","Corporama","grub-client","ia_archiver-web.archive.org","k2spider","libwww","wget","5erue","adequat","adequat-systems","auramundi","coexel","ellisphere","leadbox","mention","Moreover","mytwip","opinion-tracker","proxem","score3","trendeo","vecteurplus","verticalsearch","vsw","winello","Fetch","infoseek","MSIECrawler","Offline Explorer","sitecheck.internetseer.com","Teleport","TeleportPro","WebCopier","WebStripper","Zealbot","asknread.com","omgilibot","omgili","Argus","augure","Bloodhound","ChatGPT-User","cydralspider","Download Ninja","downloadexpress","EDD","eureka","Europresse","Explore","Factiva","Fasterfox"]}
{"domain":"optonline.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"360.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"mentalfloss.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","PerplexityBot","Google-Extended","Twitterbot","FacebookBot"]}
{"domain":"uvfuns.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Baiduspider","360Spider","Sogouspider","Yisouspider","PetalBot","Bytespider"]}
{"domain":"just-eat.co.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"blacknight.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bitget.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"newegg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","ChangeDetection","008","Nutch","mauibot","semrushbot","ahrefsbot","blexbot","seo spider","Googlebot"]}
{"domain":"sberbank-tele.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"e5.sk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bidmatic.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gamer.com.tw","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vistarsagency.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"smashingmagazine.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"groupon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"dbankcloud.eu","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"maryland.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ecwid.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","008","Googlebot","AdsBot-Google","Googlebot-Image"]}
{"domain":"byd.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"webthing.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"gtranslate.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dittoparty.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"ooklaserver.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nationalpost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["AmazonAdBot","*","Googlebot-News","omgilibot","omgili"]}
{"domain":"bbox.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"networksolutions.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tripadvisor.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":25,"stanza_tokens":["Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Cohere-ai","GPTBot","Google-CloudVertexBot","Google-Extended","Img2dataset","PanguBot","PetalBot","SentiBot","YouBot","magpie-crawler","meta-externalagent","*","ChatGPT-User","ChatGPT-User/2.0","Gemini-Deep-Research","OAI-SearchBot","applebot","PerplexityBot","bingbot","Yandex"]}
{"domain":"mez.ink","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ziprecruiter.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":33,"stanza_tokens":["googlebot","Google-Extended","Google-News","Googlebot-Image/1.0","Googlebot-Video/1.0","bingbot","Bravebot","Applebot","Pinterestbot","Slackbot","LinkedInBot","Snapchat","facebookexternalhit/1.1","TelegramBot","msnbot","Slurp","WhatsApp","ChatGPT-User","OAI-SearchBot","PerplexityBot","ClaudeBot","Claude-SearchBot","OAI-AdsBot","AdsBot-Google","AdsBot-Google-Mobile","Mediapartners-Google","adidxbot","meta-externalads","GPTBot","Applebot-Extended","Meta-ExternalAgent","CCBot","*"]}
{"domain":"wolfram.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["SemrushBot","*","FacebookBot","GPTBot","Amazonbot","PetalBot","DotBot","BLEXBot","Qwantbot","CCBot","AhrefsBot","MJ12bot","YandexBot"]}
{"domain":"dynect.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","TimeoutError"]}
{"domain":"wp2p.link","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"facebook-hardware.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"bw9iawxl.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wicknetwork.pro","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"thepiratebay.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lothar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"britishairways.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ltn.com.tw","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediatoolkitbot","MonSpark"]}
{"domain":"rand.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vuejs.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"planetminecraft.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"supplyframe.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"geneanet.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"host-food.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tudelft.nl","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"canadiantire.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ptsecurity.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"tritondigital.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adobesc.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"qiniup.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"specbench.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.gm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"clinch.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tripadvisor.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":25,"stanza_tokens":["Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Cohere-ai","GPTBot","Google-CloudVertexBot","Google-Extended","Img2dataset","PanguBot","PetalBot","SentiBot","YouBot","magpie-crawler","meta-externalagent","*","ChatGPT-User","ChatGPT-User/2.0","Gemini-Deep-Research","OAI-SearchBot","applebot","PerplexityBot","bingbot","Yandex"]}
{"domain":"impact-ad.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["GoogleOther","*"]}
{"domain":"kcl.ac.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","SiteimproveBot-Crawler","FAST-WebCrawler/0.2","Yanga WorldSearch Bot v1.1/beta","slurp","Twiceler-0.9","cludo"]}
{"domain":"tradeinn.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"diigo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google*","*"]}
{"domain":"framer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"axa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"elcomercio.pe","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":28,"stanza_tokens":["*","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","wget","grub-client","k2spider","NPBot","WebReaper"]}
{"domain":"telus.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"thalia.de","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"outbrain.org","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"joom.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":[429,429,429]}
{"domain":"star4live.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"aspnetcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sony.tv","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"c-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dtvce.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fctg.cloud","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"catawiki.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ukr.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Mediapartners-Google","MJ12bot","*","facebookexternalhit","Twitterbot"]}
{"domain":"thinkific.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"box.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"rtl.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":5,"stanza_tokens":["ChatGPT-User","GPTBot","CCBot","Google-Extended","*"]}
{"domain":"rentalcars.com","fetch_ok":false,"usable":false,"reason":"other","status":405,"error":null,"attempts":[405,405]}
{"domain":"salesmartly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bulbagarden.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":14,"stanza_tokens":["PetalBot","AspiegelBot","SemrushBot","DotBot","MauiBot","MJ12bot","ClaudeBot","Bytedance","Amazonbot","AhrefsBot","googlebot","dlvr.it","gptbot","*"]}
{"domain":"godaddysites.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"uh.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Bytespider","Baiduspider"]}
{"domain":"trenitalia.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"anjuke.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["okhttp","SkyworkSpider","YisouSpider","SpaHttpPing","Go-http-client","ImagesiftBot","ClaudeBot","Ninja","GPTBot","*"]}
{"domain":"imunify360.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jetbrains.ai","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"localytics.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oneget.org","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":[503,503,503]}
{"domain":"seaart.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"swisscom.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Googlebot","Googlebot-image","Ultraseek","search.ch","*"]}
{"domain":"sina.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[404,200]}
{"domain":"kbb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","Amazonbot","CCBot","Claude-User","Claude-SearchBot","DuckAssistBot","PerplexityBot","Perplexity-User","GrokBot","Google-InspectionTool","Mediapartners-Google","Slurp","Sogou web spider","Yahoo Ad Monitoring","Tag Inspector Spider","Adsbot-Google"]}
{"domain":"ravelry.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":25,"stanza_tokens":["*","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","wget","grub-client","k2spider","NPBot","WebReaper"]}
{"domain":"pmbmonetize.live","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"d-53.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fansly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"weborama-tech.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","ECONNRESET",200]}
{"domain":"inaturalist.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"remotepc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AdsBot-Google"]}
{"domain":"opecloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cfr.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"createjs.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"setlist.fm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":28,"stanza_tokens":["UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","wget","grub-client","k2spider","NPBot","WebReaper","*"]}
{"domain":"withings.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"trueconf.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oktacdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"glm77.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"radiopotok.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"webstaurantstore.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"felenasoft.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"steamdb.info","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":23,"stanza_tokens":["*","Googlebot","Twitterbot","SemrushBot","SiteAuditBot","AhrefsBot","BLEXBot","AdsBot-Google","AdsBot-Google-Mobile","MJ12bot","dotbot","DataForSeoBot","adsbot","barkrowler","SeekportBot","AwarioRssBot","AwarioSmartBot","PetalBot","ZoomBot","Seobility","TurnitinBot","trendictionbot","serpstatbot"]}
{"domain":"uni-hamburg.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"carrefour.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["AdsBot-Google-Mobile","AdsBot-Google","AdIdxBot","*"]}
{"domain":"hku.hk","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"cnmsn.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"truthsocial.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tutsplus.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"blogspot.com.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cbs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ibtimes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["OAI-SearchBot"],"open":[],"path_restricted":["GPTBot","ChatGPT-User"]}],"stanza_token_count":59,"stanza_tokens":["*","Applebot-Extended","AliyunSecBot","Amazonbot","Arquivo-web-crawler","AwarioRssBot","AwarioSmartBot","anthropic-ai","AudigentAdBot","BLEXBot","Bytespider","Buck","CCBot","coccocbot-web","cohere-ai","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","DataForSeoBot","DuckAssistBot","FacebookBot","ImageSift","JenkersBot","Jetslide","MoodleBot","meta-externalagent","Meta-ExternalAgent","Meta-ExternalFetcher","meta-externalfetcher","MyCentralAIScraperBot","Mojeek","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PetalBot","PerplexityBot","Perplexity-User","Poseidon Research Crawler","quillbot.com","Quora-Bot","Scrapy","Seekr","SeekrBot","SeznamBot","SeznamHomepageCrawler","scalepostAI","SentiOne","TaraGroup Intelligent Bot","Timpibot","TurnitinBot","ViennaTinyBot","yacy","YandexAdditional","YandexAdditionalBot","YouBot"]}
{"domain":"hola.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["*","Mediapartners-Google","Googlebot-Image","cohere-ai","magpie-crawler","omgili","omgilibot","Perplexity-User","PerplexityBot","Applebot-Extended","Bytespider","CCBot","DuckAssistBot"]}
{"domain":"hpytimes.info","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"charter.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"zopim.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"serverbid.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"haaretz.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"brdtest.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"filmyzilla36.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"34.in-addr.arpa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"coveo.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"dler.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"starfieldtech.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"azbyka.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"berlin.de","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"acer.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"1e100.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"bluestacks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Yandex","Baiduspider","facebookexternalhit","Yeti","*"]}
{"domain":"carmax.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"lulu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["Googlebot","Applebot","Bingbot","AdIdxBot","baiduspider","msnbot","facebookexternalhit","Slurp","Yandex","rogerbot","dotbot","*"]}
{"domain":"saawsedge.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"samhsa.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"stealth.si","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"lu.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"onthemarket.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["ClaudeBot","ULSTPropertyBot","AdsBot-Google","*"]}
{"domain":"sampath.lk","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"ultradns.biz","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"1filmyfly.mov","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"inmobi-choice.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"aihelp.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ns.tr","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bustle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Applebot","Applebot-Extended","NewsWhipbot"]}
{"domain":"wallapop.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"harborfreight.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["*","Googlebot","AdsBot-Google","Googlebot-Image","Bingbot","Yandex","DuckDuckBot","FacebookBot","Twitterbot","AhrefsBot","SemrushBot","OAI-SearchBot","ChatGPT-User","PerplexityBot","Claude-SearchBot","GPTBot","ClaudeBot","CCBot"]}
{"domain":"4chan.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gao.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":3,"stanza_tokens":["bytespider","perplexitybot","*"]}
{"domain":"vietnamnet.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xxx.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"cyberfolks.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"enjoei.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"yorku.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"island.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thriftbooks.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ltguevmavv.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sendgrid.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Swiftbot"]}
{"domain":"dn.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":32,"stanza_tokens":["CCBot","ChatGPT-User","GPTBot","Google-Extended","Google-CloudVertexBot","Omgilibot","Omgili","FacebookBot","ClaudeBot","Claude-SearchBot","Claude-User","Diffbot","DuckAssistBot","PerplexityBot","Perplexity-User","cohere-ai","cohere-training-data-crawler","Meta-ExternalAgent","Meta-ExternalFetcher","Meta-WebIndexer","Timpibot","Webzio-Extended","YouBot","Amazonbot","Bytespider","anthropic-ai","OAI-SearchBot","VelenPublicWebCrawler","Gemini-Deep-Research","ICC-Crawler","MistralAI-User","*"]}
{"domain":"ugsk.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"rshb.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","Googlebot"]}
{"domain":"xataka.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":43,"stanza_tokens":["Orthogaffe","UbiCrawler","DOC","Zao","gsa-crawler","sitecheck.internetseer.com","Zealbot","AmazonAdBot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","wget","grub-client","k2spider","NPBot","WebReaper","CNCDialer","Maxthon","MJ12bot","Slurp","*","AI2Bot","ChatGLM","ChatGLM-Spider","cohere-training-data-crawler","Diffbot","OmigiliBot","PetalBot","YouBot"]}
{"domain":"superbet.ro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":42,"stanza_tokens":["*","AdsBot-Google","AdsBot-Google-Mobile","facebookexternalhit","AhrefsSiteAudit","AI2Bot","archive.org_bot","Botify","BotifyCrawler","BotifyTester","BrightEdge Crawler","claritybot","ContentKing","DeepCrawl","DotBot","ia_archiver","JetOctopus","Lumar","MJ12bot","NovaAct","OnCrawl","Operator","Optimizer","Rogerbot","RyteBot","SearchmetricsBot","SemrushBot","SemrushBot-BA","SemrushBot-CT","SemrushBot-FT","SemrushBot-OCOB","SemrushBot-SI","SemrushBot-SWA","SeobilityBot","SERankingBot","Serpstatbot","SISTRIX Crawler","SiteAuditBot","Sitebulb","SplitSignalBot","SurferBot","woorank"]}
{"domain":"ceneo.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot"],"open":[],"path_restricted":["Claude-User","anthropic-ai"]}],"stanza_token_count":152,"stanza_tokens":["*","Googlebot","MJ12bot","archive.org_bot","ShopWiki","OmniExplorer_Bot","FreeFind","BecomeBot","Nutch","Jetbot","AskJeeves","Teoma","Jeeves","WebVac","Stanford","scooter","naver","dumbot","Hatena Antenna","grub-client","grub","looksmart","WebZip","larbin","b2w/0.1","Copernic","psbot","Python-urllib","NetMechanic","URL_Spider_Pro","CherryPicker","EmailCollector","EmailSiphon","WebBandit","EmailWolf","ExtractorPro","CopyRightCheck","Crescent","SiteSnagger","ProWebWalker","CheeseBot","LNSpiderguy","Alexibot","Stanford Comp Sci","MIIxpc","Telesoft","Website Quester","moget/2.1","WebZip/4.0","WebStripper","WebSauger","WebCopier","NetAnts","Mister PiX","WebAuto","TheNomad","WWW-Collector-E","RMA","libWeb/clsHTTP","asterias","httplib","turingos","spanner","InfoNaviRobot","Harvest/1.5","Bullseye/1.0","Crescent Internet ToolPak HTTP OLE Control v.1.0","CherryPickerSE/1.0","CherryPickerElite/1.0","WebBandit/3.50","NICErsPRO","Microsoft URL Control - 5.01.4511","DittoSpyder","Foobot","WebmasterWorldForumBot","SpankBot","BotALot","lwp-trivial/1.34","lwp-trivial","http://www.WebmasterWorld.com bot","BunnySlippers","Microsoft URL Control - 6.00.8169","URLy Warning","Wget/1.6","Wget/1.5.3","Wget","LinkWalker","cosmos","moget","hloader","humanlinks","LinkextractorPro","Offline Explorer","Mata Hari","LexiBot","Web Image Collector","The Intraformant","True_Robot/1.0","True_Robot","BlowFish/1.0"]}
{"domain":"trpcdn.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ENOTFOUND"]}
{"domain":"cloudwaysapps.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"insider.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":13,"stanza_tokens":["*","AmazonAdBot","anthropic-ai","AwarioSmartBot","Bytespider","CCBot","ClaudeBot","DataForSeoBot","Diffbot","newspaper","peer39_crawler","Scrapy","Timpibot"]}
{"domain":"google.sk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"golang.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kit.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","W3C-checklink"]}
{"domain":"raspberrypi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"stripecdn.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"parentsquare.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","facebookexternalhit"]}
{"domain":"gigya.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNRESET"]}
{"domain":"genially.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"supabase.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","GPTBot","ClaudeBot","Google-Extended","PerplexityBot"]}
{"domain":"pinterest.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":315,"stanza_tokens":["2checkout","AASA-Bot","AccessibleWebBot","Adagiobot","adbeat_bot","ADmantX","AdometryBot","Adsense-Snapshot-Google","AdsTxtCrawler","Adyen","AhrefsBot","AhrefsSiteAudit","Akismet","Algolia Crawler","AmazonProductDiscovery","Aolbot-News","APIs-Google","AppInsights","Applebot","AppleNewsBot","Archive-It","archive.org_bot","ArchiveBot","ArchiveTeam-ArchiveBot","AspiegelBot","Audisto Crawler","Awario","Baidu-ADS","Baiduspider","Barkrowler","bingbot","BingPreview","bitlybot","bl.uk_ldfc_bot","BLP_bbot","bordermanager","botify","BrandVerity","Bravebot","BrightEdge Crawler","BrightEdge-Crawler","BUbiNG","CaliperBot","CapitalOneBot","Catchpoint","cf-qualys-scanner","Chrome-Lighthouse","CJBot","ClarityBot","Clickagy Intelligence Bot v2","Cloudflare-AI-Search","CloudflareBrowserRenderingCrawler","coccocbot","ContextualBot","CookieReports","Coveobot","CriteoBot","curly-","cXensebot","Datadog","DataForSeoBot","Daum","demandbase","Discordbot","DotBot","DoubleVerifyBot","Doximity-Pipeline","Dragonbot","Dratabot","DuckAssistBot","DuckDuckBot","DuckDuckGo-Favicons-Bot","DVBot","eContext","ecurly Webhook","EngageBDR","europarchive.org","evidon","ExaleadCloudview","FacebookBot","facebookcatalog","facebookexternalhit","Facebot","FastDAST","FeedBurner","Feedfetcher-Google","FlipboardProxy","FlippBot","Freshping","GeedoProductSearch","Google Favicon","Google News","Google Page Speed Insights","Google Wireless Transcoder","Google-Apps-Script","Google-Certificates-Bridge","Google-CWS","Google-Display-Ads-Bot","Google-Firebase","Google-HotelAdsVerifier"]}
{"domain":"sber.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ECONNRESET"]}
{"domain":"timeshighereducation.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"francetvinfo.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":26,"stanza_tokens":["*","Googlebot-Image","Googlebot-News","Flipboard","Amazonbot","Applebot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","FacebookBot","GPTBot","Google-Extended","Meta-ExternalAgent","MistralAI-User","OAI-SearchBot","Perplexity-User","PerplexityBot","anthropic-ai","cohere-ai","cohere-training-data-crawler","AhrefsBot"]}
{"domain":"yandex.uz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","yandex","Twitterbot","Applebot"]}
{"domain":"boingboing.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uchi.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"temporary.site","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"vodacom.co.za","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"credit-agricole.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["NePHPila","Mediapartners-Google","*"]}
{"domain":"goneo.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mydlink.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"iplanet.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ufanet.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"alibaba-inc.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"idc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"liveperson.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"infura.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.gy","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"obsidian.md","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"er.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jiosaavn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"poolbinance.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"qzz.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"bidster.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"komoot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["FAST Enterprise Crawler","FAST-WebCrawler","barkrowler","rogerbot","dotbot","DataForSeoBot","BLEXBot","AwarioRssBot","AwarioSmartBot","MJ12bot","ZoomBot","CCBot","Bytespider","*"]}
{"domain":"brivo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"louisvuitton.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"fabulous.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"bigolive.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","GPTBot","CCBot","Google-Extended","anthropic-ai"]}
{"domain":"travelaudience.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"evidon.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"golux.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"bmwgroup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"inf0.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rtactivate.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"huaweicloudwaf.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","ENOTFOUND"]}
{"domain":"karresults.nic.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cdnhwc7.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tracker.gg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AmazonAdBot","bingbot","Yandex","ias_crawler","weborama-fetcher"]}
{"domain":"degruyter.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":6,"stanza_tokens":["*","Google-Extended","GPTBot","CCBot","FacebookBot","Omgilibot"]}
{"domain":"news-medical.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":[],"path_restricted":["GPTBot","OAI-SearchBot"]}],"stanza_token_count":2,"stanza_tokens":["*","ChatGPT-User"]}
{"domain":"edgecdn.world","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"scielo.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"momoshop.com.tw","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot-image"]}
{"domain":"blackboard.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"corel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","SEOkicks","Mauibot","ZumBot"]}
{"domain":"vdsina.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"paymaya.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"atol.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"aircanada.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"xinnet.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"asahi-net.or.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"tv2.no","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"musescore.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","BingPreview"]}
{"domain":"real.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yaplakal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"apachetutor.org","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNREFUSED"]}
{"domain":"vinted.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":7,"stanza_tokens":["*","grapeshot","GPTBot","ClaudeBot","Google-Extended","Amazonbot","CCBot"]}
{"domain":"immowelt.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","twiceler","NerdByNature.Bot","AhrefsBot"]}
{"domain":"esri.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xlgmedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thoughtco.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"nominaltechno.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"blueconic.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"baseball-reference.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":15,"stanza_tokens":["AhrefsBot","AhrefsBot/5.0","GPTBot","*","Teoma","Mediapartners-Google","Skynet","The-Matrix","Matrix","HAL9000","GroundControl","Ground-Control","Carmine","Caesar","TCBear"]}
{"domain":"respawn.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"bose.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"basf.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","GPTBot","Google-Extended","PerplexityBot"]}
{"domain":"rtl.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":22,"stanza_tokens":["AI2Bot","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","DeepSeek","Diffbot","Google-Extended","GPTBot","magpie-crawler","omgili","omgilibot","PerplexityBot","Google-CloudVertexBot","meta-externalagent","meta-externalfetcher","*"]}
{"domain":"paris.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"obihai.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pagesix.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":57,"stanza_tokens":["*","Bytespider","CCBot","claude-searchbot","claude-user","Claude-Web","ClaudeBot","FacebookBot","Google-Extended","MJ12bot","NewsNow","Perplexity-ai","PerplexityBot","PiplBot","anthropic-ai","cohere-ai","ia_archiver","magpie-crawler","news-please","omgili","omgilibot","meta-externalagent","meta-externalfetcher","A6-Indexer","AhrefsBot","Amazon-kendra","Amazon-qbusiness","amazonbot","amzn-searchbot","amzn-user","Applebot-Extended","archive.org_bot","AwarioRssBot","AwarioSmartBot","BLP_bbot","claritybot","DataForSeoBot","Diffbot","EasouSpider","ecoResearch","FriendlyCrawler","Genieo","ImagesiftBot","Meltwater","Meta-ExternalAgent","peer39_crawler","peer39_crawler/1.0","PetalBot","psbot","R6_CommentReader","Scrapy","Seekr","Spinn3r","TurnitinBot","WebVac","WebZip","FontReport"]}
{"domain":"athemes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"flippingbook.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","Googlebot","*"]}
{"domain":"defensx.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"skechers.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"symfony.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Linguee","*"]}
{"domain":"codeplex.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"cometchat.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"1tv.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"heroku.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wolframalpha.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":4,"stanza_tokens":["*","ChatGPT-User","GPTBot","Google-Extended"]}
{"domain":"sgnic.sg","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"telekom.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"1rpc.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"neverssl.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UND_ERR_SOCKET","attempts":["UND_ERR_SOCKET","UND_ERR_SOCKET"]}
{"domain":"mango-office.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oreillyauto.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"github.blog","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"afterpay.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lbl.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sankei.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":52,"stanza_tokens":["*","AI2Bot","anthropic-ai","Applebot-Extended","Arquivo-web-crawler","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","coccocbot-web","cohere-ai","cohere-training-data-crawler","Crawlspace","Diffbot","FacebookBot","FriendlyCrawler","Gensparkbot","Google-CloudVertexBot","Google-Extended","GPTBot","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","magpie-crawler","Meta-ExternalAgent","Meta-ExternalFetcher","Netvibes","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","Perplexity-ai","Perplexity-User","PetalBot","Scrapy","SemrushBot","Timpibot","Turnitin","TurnitinBot","VelenPublicWebCrawler","Webzio-Extended","yacy","yacybot","Yandex","YandexAdditional","YandexAdditionalBot","YouBot"]}
{"domain":"autoscout24.de","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"linternaute.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["Mediapartners-Google*","AmazonAdBot","*","trendkite-akashic-crawler","dotbot","AhrefsBot","DataForSeoBot","SemrushBot"]}
{"domain":"mypurecloud.com","fetch_ok":false,"usable":false,"reason":"other","status":500,"error":null,"attempts":[500,500,500]}
{"domain":"prudential.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":14,"stanza_tokens":["Twitterbot","OAI-SearchBot","ChatGPT-User","PerplexityBot","FirecrawlAgent","AndiBot","ExaBot","PhindBot","YouBot","facebookexternalhit","GPTBot","CCBot","Google-Extended","*"]}
{"domain":"wirtgen-group.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Baiduspider","*"]}
{"domain":"google.com.eg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"rtl.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":4,"stanza_tokens":["*","GPTBot","Google-Extended","CCBot"]}
{"domain":"covers.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"menards.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lsu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["SemrushBot","*"]}
{"domain":"ipsos.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"taoa.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"pik.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","Googlebot"]}
{"domain":"suntimes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":39,"stanza_tokens":["*","Googlebot","Googlebot-News","GoogleOther-Image","GoogleOther-Video","Bingbot","DuckDuckBot","Slurp","Applebot","facebookexternalhit","Meta-ExternalAgent","Meta-ExternalFetcher","SmartNews-AdsBot","SmartNewsBot","Twitterbot","Applebot-Extended","Bytespider","CCBot","Claude-Web","ClaudeBot","Diffbot","FriendlyCrawler","GPTBot","Google-Extended","GoogleOther","ICC-Crawler","ImagesiftBot","PerplexityBot","PetalBot","Scrapy","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot","anthropic-ai","cohere-ai","img2dataset","omgili","omgilibot"]}
{"domain":"1cmycloud.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"americanbar.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pixeldrain.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"denverpost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":68,"stanza_tokens":["*","AI2Bot","Ai2Bot-Dolma","Amazonbot","Amzn-SearchBot","anthropic-ai","Applebot","Applebot-Extended","Archive.org_bot","Baiduspider","Brightbot 1.0","Bytespider","carbon-umbrella-bot","CCBot","ChatGLM-Spider","ChatGPT-User","ClaudeBot","Claude-Searchbot","Claude-User","Claude-Web","Cohere-training-data-crawler","Cotoyogi","Dalvik","Datenbank Crawler","Diffbot","DocSpotBot","DuckAssistBot","FacebookBot","GnowitNewsbot","Google-Extended","GPTBot","grok-crawler","ia_archiver-web.archive.org","ICC-Crawler","imageSpider","Kangaroo Bot","laion-huggingface-processor","LCC","Mantisbot","Meta-ExternalAgent","MistralAI-User","netEstate Imprint Crawler","OAI-SearchBot","omgili","PanguBot","PerplexityBot","Perplexity-User","PetalBot","SBIntuitionsBot","SirdataBot","Spider","Thinkbot","Timpibot","VelenPublicWebCrawler","Vivaldi","Webzio-extended","YandexBot","YouBot","AmazonAdBot","facebookexternalhit","grapeshot","Mozilla/5.0 (compatible; EchoboxBot/1.0; hash/w4mwnpbXf3MFAbxOkJRw; +http://www.echobox.com)","Twitterbot","Mozilla/5.0 (compatible; NewsRoom.BI/0.1; +http://www.newsroom.bi/bot.html)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36(KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; mrfCompass-Marshall/1.0)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Jukebox/1.0)"]}
{"domain":"iaea.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"empik.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","AdsBot-Google","AdsBot-Mobile-Google","Bingbot","AdIdxBot"]}
{"domain":"outboundproxy.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"uptodown.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cronolog.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"stabletransit.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"goo.ne.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kameleoon.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"medrxiv.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rbc.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jagex.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"torrid.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["Pinterestbot","FacebookBot","facebookexternalhit/1.1","facebookexternalhit/1.0","bingbot","Googlebot","Googlebot-Image","AdsBot-Google-Mobile","AdsBot-Google","AdsBot-Google-Mobile-Apps","*"]}
{"domain":"cam4.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"l.ink","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"hostedrmm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"inquirer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"goldprice.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AmazonAdBot","*"]}
{"domain":"thekitchn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","grapeshot"]}
{"domain":"razerzone.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","CazoodleBot","MJ12bot","dotbot/1.0","Gigabot","Mediapartners-Google","AdsBot-Google","Googlebot-Image","Googlebot","Bingbot","Baiduspider","GPTBot","PerplexityBot","ClaudeBot","Google-Extended","YandexBot","Qwantify","SeznamBot","Yeti","Slurp"]}
{"domain":"jbhifi.com.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","adsbot-google","Nutch","AhrefsBot","AhrefsSiteAudit","MJ12bot","Pinterest"]}
{"domain":"centrum.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"clemson.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","googlebot"]}
{"domain":"combankdigital.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"sbnation.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":["GPTBot"],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":105,"stanza_tokens":["Amazonbot","Applebot","GPTBot","A6-Indexer","AhrefsBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot-Extended","Baiduspider","betaBot","BLEXBot","botify","brands-bot-logo","Brightbot","Bytespider","calculon","CCBot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","collection@infegy.com","Cotoyogi","Crawl4AI","CrunchBot","deepcrawl","DeepSeekBot","Diffbot","DirBuster","DTS","DuckAssistBot","EasouSpider","FacebookBot","FirecrawlAgent","FriendlyCrawler","GlutenCrawler","Google-CloudVertexBot","Google-Extended","GrapeshotCrawler","iaskspider","ICC-Crawler","ImagesiftBot","img2dataset","JackCrawl","Jetslide","linkapedia_bot","Mediatoolkitbot","MegaIndex","meta-externalagent","meta-externalfetcher","Microsoft BITS","Microsoft URL Control","Microsoft-WebDAV-MiniRedir","MistralAI-User","news-please","Novellum","NTENTbot","Nutch","omgili","Omgilibot","OpenLinkProfiler","Perplexity-User","PerplexityBot","PetalBot","Primalbot","python-requests","Re-Animator","Riddler","SafeDNSBot","SBIntuitionsBot","Scrapy","SearchAtlas","SemanticScholarBot","SemrushBot","SentiBot","Sogou","Spider-Pig","sqlmap","StorygizeBot","Test Certificate Info","TikTokSpider","Timpibot","Turnitin","uipbot","vBSEO","VegeBot","VelenPublicWebCrawler","Web Link Validator","WebCruiser"]}
{"domain":"guinnessworldrecords.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hexun.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","SemrushBot","DotBot","BLP_bbot","YandexBot","archive.org_bot"]}
{"domain":"tmweb.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UND_ERR_HEADERS_OVERFLOW","attempts":["UND_ERR_HEADERS_OVERFLOW","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"vidio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-56.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"opaque.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"logmein.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"apkmirror.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ws-hp.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zoopla.co.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"cimri.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":17,"stanza_tokens":["*","Yandex","FacebookBot","GrapeshotCrawler","BLEXBot","AdsrvrBot","SemrushBot","PingdomBot","MJ12bot","Twiceler","dotbot","Sogou","Baidu","Baiduspider","YandexMarket","yaanibot","Adsbot-Google"]}
{"domain":"oregonlive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","ia_archiver","archive.org_bot","heritrix","My-heritrix-crawler","Timpibot","omgili","Bytespider","DuckAssistBot","Applebot-Extended","Google-Extended","Google-CloudVertexBot","GoogleOther","GPTBot","CCBot","Amazonbot","FacebookBot","cohere-ai","cohere-training-data-crawler","Omgilibot","ClaudeBot","Diffbot","PanguBot","MistralAI-User","webz.io","DeepSeekBot","AmazonAdBot","Amzn-SearchBot","Amzn-User","Claude-SearchBot","Claude-User","PetalBot","Meta-ExternalFetcher","Meta-WebIndexer","OAI-SearchBot","ChatGPT-User","PerplexityBot","Perplexity-User"]}
{"domain":"buyee.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pia.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"radissonhotels.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tlivepush.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"digikey.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":19,"stanza_tokens":["*","AdsBot-Google","AdsBot-Google-Mobile","AdIdxBot","Google-Extended","Gemini-Deep-Research","GPTBot","ChatGPT-User","OAI-SearchBot","PerplexityBot","Perplexity-User","ClaudeBot","Claude-User","Claude-SearchBot","AhrefsBot","dotbot","MJ12bot","PetalBot","Yandex"]}
{"domain":"maharashtra.gov.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"matomo.cloud","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"google.com.ni","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"hs.fi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":46,"stanza_tokens":["*","ChatGPT-user","CCBot","GPTBot","Google-Extended","ImagesiftBot","FacebookBot","Omgilibot","Amazonbot","anthropic-ai","Claude-Web","cohere-ai","omgili","PerplexityBot","YouBot","Bytespider","Diffbot","Applebot-Extended","OAI-SearchBot","ClaudeBot","Google-CloudVertexBot","Meta-ExternalAgent","Meta-ExternalFetcher","Quora-Bot","DataForSeoBot","Webzio-Extended","FriendlyCrawler","ChatGPT-User","meta-externalagent","meta-externalfetcher","TeleportPro","Qwam content intelligence","Timpibot","Kangaroo Bot","AI2Bot","VelenPublicWebCrawler","img2dataset","Sidetrade indexer bot","PanguBot","ISSCyberRiskCrawler","Ai2Bot-Dolma","Claude-SearchBot","Claude-User","MistralAI-User","DeepSeekBot","Perplexity-User"]}
{"domain":"wpscdn.cn","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"surriel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"4.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"medtronic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xgroovy.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vultr.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"correiobraziliense.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["facebookexternalhit","AmazonAdBot","*","Googlebot-News","Google-Extended"]}
{"domain":"sportingnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":54,"stanza_tokens":["*","Googlebot","Bingbot","Slurp","DuckDuckBot","OAI-SearchBot","PerplexityBot","ChatGPT-User","Claude-Web","AmazonAdBot","Amazonbot","anthropic-ai","Applebot-Extended","archive.org_bot","Arquivo-web-crawler","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","DataForSeoBot","Diffbot","DuckAssistBot","europarchive.org","FacebookBot","FriendlyCrawler","Google-CloudVertexBot","heritrix","ia_archiver","ia_archiver-web.archive.org","ImagesiftBot","LinkArchiver","magpie-crawler","Meta-ExternalAgent","meta-externalagent","Meta-ExternalFetcher","meta-externalfetcher","news-please","Nicecrawler","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","Primalbot","Quora-Bot","Scrapy","Timpibot","TurnitinBot","web-archive-net.com.bot","YouBot","GPTBot","Google-Extended"]}
{"domain":"dnb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","ScoutJet","Twitterbot","GPTBot","Applebot"]}
{"domain":"sportbox.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"asm.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"orange.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot-image"]}
{"domain":"yieldlab.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"cloudbirds.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"capgemini.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"azcentral.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":287,"stanza_tokens":["anthropic-ai","AmazonBot","Claude","Claude-Web","claudeweb","Amzn-SearchBot","Amzn-User","AndiBot","cohere-ai","ClaudeBot","ChatGPT-User","Googlebot-News","GPTBot","CCBot","Google-Extended","DataForSeoBot","Diffbot","Diffbot-user","Kagibot","FacebookBot","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Scrapy","TurnitinBot","Applebot-Extended","Meta-ExternalAgent","meta-externalagent","OAI-SearchBot","Bytespider","Claude-User","Claude-SearchBot","AI2Bot","AhrefsBot","Ai2Bot-Dolma","Applebot","Arquivo-web-crawler","AspiegelBot","AwarioRssBot","AwarioSmartBot","Barkrowler","coccocbot-web","cohere-training-data-crawler","crawler4j","Crawlspace","dotbot","DuckAssistBot","FriendlyCrawler","Google-CloudVertexBot","https://hada.news","https://www.imediaethics.org","iaskspider/2.0","ICC-Crawler","icc-crawler","ICC Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","LivelapBot","MauiBot","Meltwater","Meta-ExternalFetcher","MJ12bot","Mojeek","MojeekBot","MoodleBot","NeticleBot","Netvibes","PanguBot","PetalBot","SeekportBot","Seekr","SemrushBot","SeznamBot","Sidetrade Indexer Bot","Timpibot","Turnitin","VelenPublicWebCrawler","Webzio-Extended","yacy","yacybot","Yandex","YandexAdditional","YandexAdditionalBot","YandexBot","YandexImages","YandexNews","Baiduspider","Baiduspider-image","BaiduSpider-render","Baiduspider-video","Sogou Web Spider","yisouspider"]}
{"domain":"blogspot.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"medicare.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hola.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["GPTBot","PerplexityBot","Google-Extended","anthropic-ai","Claude-Web","*"]}
{"domain":"ebird.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":60,"stanza_tokens":["bingbot","*","FacebookExternalHit","AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","Brightbot 1.0","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Diffbot","DuckAssistBot","FacebookBot","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","imgproxy","ISSCyberRiskCrawler","Kangaroo Bot","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","NovaAct","OAI-SearchBot","omgili","omgilibot","Operator","PanguBot","Perplexity-User","PerplexityBot","PetalBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","TikTokSpider","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot"]}
{"domain":"agenziaentrate.gov.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"r-project.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"prosv.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","Googlebot"]}
{"domain":"python.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["NPBot-*"]}
{"domain":"substackcdn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"honda.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"netperf.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"xhamster3.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"uni-erlangen.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"h264.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"codecanyon.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Pinterest","CCBot"]}
{"domain":"bstats.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vzbi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"vidazoo.services","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"musical.ly","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNRESET"]}
{"domain":"paddypower.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mtu.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"mangadex.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"overdrive.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"crushon.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ku.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"imagemagick.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eiga.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":5,"stanza_tokens":["*","Megalodon","GPTBot","ClaudeBot","Bytespider"]}
{"domain":"hoyolab.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kolesa.kz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"afr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":52,"stanza_tokens":["*","Google-CloudVertexBot","Google-Extended","ChatGPT-User","GPTBot","OAI-SearchBot","OAISearch","anthropic-ai","claude-web","claudebot","facebookbot","meta-externalagent","meta-externalfetcher","applebot-extended","perplexitybot","cohere-ai","youbot","amazonbot","aliyunsecbot","audigentadbot","awariorssbot","awariosmartbot","blexbot","bytespider","ccbot","dataforseobot","diffbot","duckassistbot","echoboxbot","friendlycrawler","ia_archiver","imagesiftbot","mycentralaiscraperbot","newsnow","news-please","omgili","omgilibot","webzio-extended","peer39_crawler","peer39_crawler/1.0","quillbot.com","quora-bot","scrapy","seekrbot","seznamhomepagecrawler","taragroup intelligent bot","timpibot","turnitinbot","viennatinybot","jetslide","magpie-crawler","poseidon research crawler"]}
{"domain":"crcdn.pro","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fotocasa.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":41,"stanza_tokens":["Mediapartners-Google","Google-Display-Ads-Bot","AdsBot-Google","AdsBot-Google-Mobile","TwitterBot","Adidum","Bloglines/3.1","DOC","Download Ninja","Exabot","Fetch","HTTrack","Jyxobot/1","Mail.Ru","Microsoft.URL.Control","NPBot","Offline Explorer","SiteSnagger","Speedy","Teleport","TeleportPro","UbiCrawler","WebCopier","WebReaper","WebStripper","WebZIP","Xenu","Zao","Zealbot","ZyBORG","cityreview","dotbot","e-SocietyRobot","grub-client","larbin","libwww","linko","psbot","sitecheck.internetseer.com","wget","*"]}
{"domain":"speedtestmark.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fhvfd.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"arkoselabs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Bytespider","SiteAuditBot"]}
{"domain":"vmailru.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"takealot.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"avant.si","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"sonnenbatterie.de","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ximalaya.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"byspotify.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gingerall.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"vinted.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":7,"stanza_tokens":["*","grapeshot","GPTBot","ClaudeBot","Google-Extended","Amazonbot","CCBot"]}
{"domain":"okjds8z.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"storygize.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"spoonflower.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"tripo3d.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Baiduspider"]}
{"domain":"google.com.mm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"goindigo.in","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"goxh.today","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"adtarget.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"acsbapp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bgchprod.info","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENETUNREACH"]}
{"domain":"certum.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thermofisher.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"142.in-addr.arpa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"corpinter.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"crypto.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["waybackurls","AdsBot-Google","Googlebot","AdsBot-Google-Mobile","*"]}
{"domain":"nspk.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_VERIFY_LEAF_SIGNATURE","attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE","UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"falabella.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"9gag.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"shell.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["SiteimproveBot","SiteimproveBot-Crawler","*"]}
{"domain":"ibosscloud.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"sardinasystems.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"pantip.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"iol.pt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","OpenAI","ChatGPT-User","Google-Extended","OpenAI-Crawler","GPTBot","OAI-SearchBot"]}
{"domain":"cricketwireless.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"samsungmobile.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"tenjin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","GPTBot","ChatGPT-User","OAI-SearchBot","Google-Extended","PerplexityBot","ClaudeBot","Applebot-Extended","Meta-ExternalAgent","Meta-ExternalFetcher","CCBot","Amazonbot","cohere-ai","Omgilibot","Omgili"]}
{"domain":"avid.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"server-k2cz.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"360qhcdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"EAI_AGAIN","attempts":["EAI_AGAIN","TimeoutError"]}
{"domain":"belgium.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"itglue.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"progressive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AdsBot-Google","AdsBot-Google-Mobile"]}
{"domain":"hud.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"itunes.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"td.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","AdsBot-Google","Googlebot","Googlebot-Image"]}
{"domain":"informer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hani.co.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":7,"stanza_tokens":["*","GPTBot","Bytespider","Amazonbot","ClaudeBot","BLP_bbot","Slackbot"]}
{"domain":"fwupd.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ebscohost.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"bell.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","AdsBot-Google","Googlebot","Googlebot-Image"]}
{"domain":"mediamarkt.de","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"yesstyle.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"clientgear.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_SSL_WRONG_SIGNATURE_TYPE"]}
{"domain":"sony.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Baiduspider","BaiduImagespider","BaiduMobaider","Yeti","Googlebot","*"]}
{"domain":"homeoffice.gov.uk","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"draftkings.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"getdropbox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Twitterbot","facebookexternalhit","*"]}
{"domain":"sbs.com.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":13,"stanza_tokens":["*","GPTBot","NewsNow","CCBot","TurnitinBot","PetalBot","MoodleBot","BLEXBot","R6_CommentReader","ICC-Crawler","TinEye-bot","Nutch","trendkite-akashic-crawler"]}
{"domain":"manageengine.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["*","OAI-SearchBot","GPTBot","ChatGPT Agent","ChatGPT-User","ClaudeBot","Claude-Web","Claude-User","Claude-SearchBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GoogleAgent-Mariner","Google-CloudVertexBot","CloudVertexBot","Meta-ExternalAgent","Meta-ExternalFetcher","meta-externalagent","meta-externalfetcher","PerplexityBot","Perplexity-User","Applebot","Applebot-Extended","MistralAI-User","MistralAI-User/1.0","cohere-ai","cohere-training-data-crawler","YouBot","PhindBot","Amazonbot","Diffbot","Bytespider","CCBot"]}
{"domain":"ovh.us","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":6,"stanza_tokens":["*","GPTBot","ChatGPT-User","CCBot","anthropic-ai","TextBulkerBot"]}
{"domain":"invisalign.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"weatherkit.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ECONNRESET"]}
{"domain":"giga.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":12,"stanza_tokens":["*","AmazonAdBot","GPTBot","PerplexityBot","ClaudeBot","peer39_crawler","Applebot-Extended","Bytespider","CCBot","Google-Extended","Meta-ExternalAgent","meta-externalagent"]}
{"domain":"freetds.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"greenend.org.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kuleuven.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aktuality.sk","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":21,"stanza_tokens":["Amazonbot","Cohere-AI","Bytespider","cohere-ai","Ai2Bot-Dolma","Timpibot","Omgilibot","Mistral AI-User","Amzn-User","diffbot-user","Amzn-SearchBot","DuckAssistBot","YouBot","seodiffbot","PetalBot","img2dataset","omgili","Scrapy","DeepSeekBot","Sogou web spider","Baiduspider"]}
{"domain":"google.co.cr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"ix.ru","fetch_ok":true,"usable":false,"reason":"fetch_error","status":200,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bedrock.tech","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"govdelivery.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","OAI-SearchBot","GPTBot","ChatGPT-User","ClaudeBot","PerplexityBot","FirecrawlAgent","AndiBot","ExaBot","PhindBot","YouBot"]}
{"domain":"actionverb.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"cmd.su","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"p-n.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bsky.network","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"treasuredata.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["*","RavenCrawler","rogerbot","dotbot","SemrushBot","SiteAuditBot","SplitSignalBot","PowerMapper","Swiftbot","lyticsbot","Dubbotbot","PopeTech-CrawlBot","PopeTech-ScanBot"]}
{"domain":"milliyet.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":12,"stanza_tokens":["*","AmazonAdBot","anthropic-ai","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","GPTBot","OAI-SearchBot","quillbot.com","PerplexityBot","TurnitinBot"]}
{"domain":"make.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"shb-sync.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"leeds.ac.uk","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"planalto.gov.br","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ECONNRESET"]}
{"domain":"springeropen.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[404,200]}
{"domain":"ionos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Slurp","msnbot","dotbot","kinshoo"]}
{"domain":"snowflakecomputing.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"band.us","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":[],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot"],"open":[],"path_restricted":["Claude-User","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":13,"stanza_tokens":["*","AhrefsBot","Semrushbot","Yeti","GPTBot","OAI-SearchBot","PerplexityBot","Google-Extended","ClaudeBot","Claude-SearchBot","meta-externalagent","Applebot-Extended","CCBot"]}
{"domain":"bytens.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"skool.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"telkomsel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rug.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["MagnetmeBot","*"]}
{"domain":"713mtauburnctcolumbusoh43085.st","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"beacons.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dbs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":140,"stanza_tokens":["Googlebot","*","OmniExplorer-Bot","FreeFind","BecomeBot","Nutch","Jetbot-1","Jetbot","WebVac","Stanford","naver","dumbot","Hatena-Antenna","grub-client","grub","WebZip","larbin","b2w-01","Copernic","psbot","Python-urllib","NetMechanic","URL_Spider_Pro","CherryPicker","EmailCollector","EmailSiphon","WebBandit","EmailWolf","ExtractorPro","CopyRightCheck","Crescent","SiteSnagger","ProWebWalker","CheeseBot","LNSpiderguy","Alexibot","Teleport","TeleportPro","Stanford-Comp-Sci","MIIxpc","Telesoft","Website-Quester","moget-2-1","WebZip-4-0","WebStripper","WebSauger","WebCopier","NetAnts","Mister-PiX","WebAuto","TheNomad","WWW-Collector-E","RMA","libWeb-clsHTTP","asterias","httplib","turingos","spanner","InfoNaviRobot","Harvest-1.5","Bullseye-1.0","Crescent-Internet-ToolPak-HTTP-OLE-Control-v-1-0","CherryPickerSE-1-0","CherryPickerElite-1-0","WebBandit-3-50","NICErsPRO","Microsoft-URL-Control-5-01-4511","DittoSpyder","Foobot","SpankBot","BotALot","lwp-trivial-1-34","lwp-trivial","BunnySlippers","Microsoft-URL-Control-6-00-8169","URLy-Warning","Wget-1-6","Wget-1-5-3","Wget","LinkWalker","cosmos","moget","hloader","humanlinks","LinkextractorPro","Offline-Explorer","Mata-Hari","LexiBot","Web-Image-Collector","The-Intraformant","True_Robot-1-0","True_Robot","BlowFish-1-0","JennyBot","MIIxpc-4-2","BuiltBotTough","ProPowerBot-2-14","BackDoorBot-1-0","toCrawl-UrlDispatcher","WebEnhancer"]}
{"domain":"tapclap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"so-net.ne.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"brand-display.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"sporcle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":26,"stanza_tokens":["HubSpot","Bytespider","GPTBot","claudebot","panscient","PerplexityBot","ImagesiftBot","Amazonbot","Applebot-Extended","CCBot","Diffbot","FacebookBot","omgili","carbon-umbrella-bot","meta-externalagent","Opebot-v","OAI-SearchBot","ChatGPT-User","TaraGroup Intelligent Bot","SynthesiBot","AliyunSecBot","Baiduspider","Brightbot","Thinkbot","Sogou web spider","*"]}
{"domain":"copyright.gov","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"verkada-lan.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"tvn24.pl","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"documentcloud.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"mako.co.il","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"betano.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"arvancdn.ir","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","attempts":["ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"pro32connect.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nic.live","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"inss.gov.br","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"komect.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[403,200]}
{"domain":"ultradns2.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"liverpool.com.mx","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"louvre.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","GPTBot","ChatGPT-User","anthropic-ai","ClaudeBot","CCBot","Amazonbot","Bytespider","Applebot","Sogou web spider","Google-Extended"]}
{"domain":"cio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":37,"stanza_tokens":["*","Ai2Bot-Dolma","AI2Bot","anthropic-ai","Applebot-Extended","Arquivo-web-crawler","AwarioRssBot","AwarioSmartBot","Barkrowler","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","crawler4j","dotbot","Diffbot","FacebookBot","FriendlyCrawler","GPTBot","https://hada.news","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","MJ12bot","PanguBot","Scrapy","Seekr","Timpibot","VelenPublicWebCrawler","Webzio-Extended"]}
{"domain":"ltmsphrcl.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ovhnextmillmedia.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nyt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":62,"stanza_tokens":["*","Googlebot","AliyunSecBot","AmazonBot","anthropic-ai","Applebot-Extended","archive.org_bot","AudigentAdBot","AwarioRssBot","AwarioSmartBot","BLEXBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","DuckAssistBot","EchoboxBot","FacebookBot","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","GPTBot","ImagesiftBot","Jetslide","magpie-crawler","Meta-ExternalAgent","meta-externalagent","Meta-ExternalFetcher","meta-externalfetcher","Meta-WebIndexer","meta-webindexer/1.1","MyCentralAIScraperBot","NewsNow","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Perplexity-User","Poseidon Research Crawler","quillbot.com","Quora-Bot","Scrapy","SeekrBot","SeznamHomepageCrawler","TaraGroup Intelligent Bot","Timpibot","TurnitinBot","ViennaTinyBot","YouBot","AmazonAdBot","facebookexternalhit","Twitterbot","RedditBot"]}
{"domain":"camsoda.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uni-bonn.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["BluechipBacklinks","*","Googlebot"]}
{"domain":"unibo.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"obi.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uidapi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"miami.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kuronekoyamato.co.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"ves.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"delfi.lt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":28,"stanza_tokens":["*","AI2Bot","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Diffbot","DuckAssistBot","FacebookBot","Google-Extended","GPTBot","Kangaroo Bot","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","omgili","PanguBot","PerplexityBot","Timpibot","Webzio-Extended","YouBot"]}
{"domain":"mtv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","PetalBot"]}
{"domain":"prodigygame.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"skylinewebcams.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AwarioRssBot","AwarioSmartBot","*"]}
{"domain":"clover.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Googlebot","Googlebot-image","*"]}
{"domain":"net-line.pro","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ilmessaggero.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":21,"stanza_tokens":["*","GPTBot","Google-Extended","anthropic-ai","CCBot","Omgilibot","cohere-ai","ChatGPT-User","OAI-SearchBot","PerplexityBot","Amazonbot","YouBot","Claude-Web","Claudebot","Claude-User","Claude-SearchBot","Diffbot","Bytespider","ImagesiftBot","Meltwater","Seekr"]}
{"domain":"smartclip.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"t-ipconnect.de","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"samsara.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"securly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bandlab.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":19,"stanza_tokens":["*","Amazonbot","anthropic-ai","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","DataForSeoBot","GPTBot","Google-Extended","magpie-crawler","omgili","omgilibot","PetalBot","PerplexityBot","SemrushBot","weborama-fetcher"]}
{"domain":"ecfr.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sinsay.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Meta-ExternalAds","*"]}
{"domain":"ibscorp.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"capterra.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ig.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","AdsBot-Google","Mediapartners-Google","AmazonAdBot"]}
{"domain":"collinsdictionary.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["AmazonAdBot","*","Bingbot","msnbot","Slurp","FacebookBot","proximic","Verity/1.1","SirdataBot"]}
{"domain":"konghq.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"puzztake.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"getpostman.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"humbedooh.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"CERT_HAS_EXPIRED"]}
{"domain":"gpo.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"toyokeizai.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"motherjones.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":13,"stanza_tokens":["*","Claude-User","Claude-SearchBot","ChatGPT-User","OAI-SearchBot","Perplexity-User","Applebot-Extended","Amazonbot","anthropic-ai","GPTBot","ClaudeBot","cohere-ai","meta-externalagent"]}
{"domain":"diy.com","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":[503,503,503]}
{"domain":"castbox.fm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":15,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","Googlebot","DuckDuckBot","Twitterbot","Bingbot","Facebot"]}
{"domain":"newrepublic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["upday","AmazonAdBot","*"]}
{"domain":"liebertpub.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":8,"stanza_tokens":["*","facebookexternalhit","LinkedInBot","Twitterbot","Baiduspider","GPTBot","CCBot","Google-Extended"]}
{"domain":"anlian.co","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"buienradar.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"daraz.pk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xs4all.nl","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ca-servers.ca","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rtcfront.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ucalgary.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"akamaistream.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"xhchannel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"crcldu.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"cdn-tinkoff.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"hstgr.cloud","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"playtika.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"swagbucks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","AdsBot-Google","Mediapartners-Google","AdsBot-Google-Mobile"]}
{"domain":"meti.go.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"pointclickcare.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hankyung.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["dotbot","AhrefsBot","SemrushBot","SemrushBot-SA","SemrushBot-BA","*"]}
{"domain":"lilithgame.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"dezeen.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"workday.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"openedition.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"advmusic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pagina12.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"desjardins.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Twitterbot","FacebookBot","Elastic-Crawler","*"]}
{"domain":"transcend-cdn.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"kerala.gov.in","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"core.ac.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"apachetoday.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"appfolio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"huffingtonpost.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":24,"stanza_tokens":["*","meta-externalagent","Meta-ExternalFetcher","ChatGPT-User","OAI-SearchBot","Bytespider","DuckAssistBot","Perplexity-User","Timpibot","ClaudeBot","Amazonbot","GPTBot","PerplexityBot","CCBot","FacebookBot","Ai2Bot-Dolma","Claude-SearchBot","Claude-User","omgilibot","TtoYouBot","WebGPTBot","Verity/1.1","Scope3/2.0 (scope3.com)","AmazonAdBot"]}
{"domain":"mjedge.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ECONNREFUSED"]}
{"domain":"schoology.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"rentry.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jiji.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":65,"stanza_tokens":["*","Mediapartners-Google","Mediapartners","Googlebot-Image","Googlebot","msnbot","bingbot","Baiduspider+(+http://www.baidu.jp/spider/)","BaiduImagespider(+http://www.baidu.jp/spider/)","Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)","Mozilla/5.0 (compatible; Baiduspider/3.0; +http://www.baidu.com/search/spider.html)","Mozilla/5.0 (compatible; Baiduspider/4.0; +http://www.baidu.com/search/spider.html)","Mozilla/5.0 (compatible; Baiduspider/5.0; +http://www.baidu.com/search/spider.html)","Baiduspider/2.0","Baiduspider/3.0","Baiduspider/4.0","Baiduspider/5.0","Baiduspider/2.0+(+http://www.baidu.com/search/spider.htm)","Slurp","Yahoo! Slurp","Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/)","Y!J","popIn_Agent","Applebot","Twitterbot","Crowsnest","logly","Clipper","GrapeshotCrawler","grapeshot","Zucks Recommend Engine Crawler Bot","Opebot-v2-beta (http://www.1plusx.com)","cXensebot","Y!J-BRJ/YATS crawler","AmazonBot","anthropic-ai","AppleBot-Extended","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","cohere-training-data-crawler","Diffbot","FacebookBot","Google-Extended","GPTBot","ICC-Crawler","magpie-crawler","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","omgili","omgilibot","PerplexityBot","Perplexity-ai","Perplexity-User","SBIntuitionsBot","Scrapy","Timpibot","Webzio","Webzio-Extended"]}
{"domain":"edmunds.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":15,"stanza_tokens":["Googlebot","AdsBot-Google","grapeshot","*","Mediapartners-Google","ias_crawler","GPTBot","AI2Bot","Ai2Bot-Dolma","ClaudeBot","Bytespider","cohere-training-data-crawler","DeepSeekBot","meta-externalagent","PetalBot"]}
{"domain":"bodis.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"optable.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xunlei.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wwsga.me","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"naver.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"computrabajo.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"forthnet.gr","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","ETIMEDOUT"]}
{"domain":"desmos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"realsimple.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"domru.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"fineartamerica.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"formstack.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Twitterbot"]}
{"domain":"2cnt.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"msk.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"nieuwsblad.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":21,"stanza_tokens":["*","Googlebot-News","Amazonbot","anthropic-ai","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","Diffbot","FacebookBot","Google-Extended","GPTBot","magpie-crawler","omgili","omgilibot","PerplexityBot","Google-CloudVertexBot","meta-externalagent","meta-externalfetcher"]}
{"domain":"44.in-addr.arpa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"1337x.to","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"thenation.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nhk.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":13,"stanza_tokens":["GPTBot","ChatGPT-User","ClaudeBot","Google-Extended","Applebot-Extended","anthropic-ai","cohere-ai","CCBot","ICC-Crawler","Bytespider","OAI-SearchBot","PerplexityBot","Meta-ExternalAgent"]}
{"domain":"reliefweb.int","fetch_ok":false,"usable":false,"reason":"other","status":406,"error":null,"attempts":[406,406]}
{"domain":"amazonsilk.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"inria.fr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"grofer.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"rcdn-web.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"ezydomain.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tango.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["facebookexternalhit","meta-externalagent","*"]}
{"domain":"illumin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"leavenworth-net.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"igvita.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"grab.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"realtor.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sport.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":28,"stanza_tokens":["008","UnisterBot","JikeSpider","Zealbot","WebStripper","WebCopier","Fetch","Teleport","HTTrack","magpie-crawler","MJ12bot","DotBot","spbot","BLEXBot","BUbiNG","Yandex","Baiduspider","OAI-SearchBot","ChatGPT-User","GPTBot","ClaudeBot","Meta-ExternalAgent","PerplexityBot","Google-Extended","AmazonAdBot","*","Twitterbot","facebookexternalhit"]}
{"domain":"euractiv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"supremecourt.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["discobot","SOTScraper","*"]}
{"domain":"videoamp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"abplive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ig.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rona.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"esxdos.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"obozrevatel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AmazonAdBot","facebookexternalhit/1.1"]}
{"domain":"tiket.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mgtv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tribunnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","Googlebot","Googlebot-News","Googlebot-Image","Googlebot-Video","Google-InspectionTool","Google-NotebookLM","bingbot","BingPreview","msnbot","msnbot-media","Google-Extended","facebookexternalhit","GPTBot","SemrushBot","AhrefsBot"]}
{"domain":"jigsawplanet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":17,"stanza_tokens":["Mediapartners-Google","AdsBot-Google","AdsBot-Google-Mobile","AmazonAdBot","Googlebot-Image","CCBot","ChatGPT-User","GPTBot","Google-Extended","PerplexityBot","ClaudeBot","dotbot","Riddler","BLEXBot","AhrefsBot","MauiBot","*"]}
{"domain":"binance.info","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"teepublic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Pinterest/0.2 (+https://www.pinterest.com/bot.html)","Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)","Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)","facebookexternalhit"]}
{"domain":"tuoitre.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"blocket.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"facemojikeyboard.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"java.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"registrydns.co","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dome9.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"codebig2.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.bg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"mcldns.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"thestar.com.my","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vulture.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":["GPTBot"],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":107,"stanza_tokens":["Mediapartners-Google*","TruliaBot","Twitterbot","*","Amazonbot","Applebot","GPTBot","A6-Indexer","AhrefsBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot-Extended","Baiduspider","betaBot","BLEXBot","botify","brands-bot-logo","Brightbot","Bytespider","calculon","CCBot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","collection@infegy.com","Cotoyogi","Crawl4AI","CrunchBot","deepcrawl","DeepSeekBot","Diffbot","DirBuster","DTS","DuckAssistBot","EasouSpider","FacebookBot","FirecrawlAgent","FriendlyCrawler","GlutenCrawler","Google-CloudVertexBot","Google-Extended","GrapeshotCrawler","iaskspider","ICC-Crawler","ImagesiftBot","img2dataset","JackCrawl","Jetslide","linkapedia_bot","Mediatoolkitbot","MegaIndex","meta-externalagent","meta-externalfetcher","Microsoft BITS","Microsoft URL Control","Microsoft-WebDAV-MiniRedir","MistralAI-User","news-please","Novellum","NTENTbot","Nutch","omgili","Omgilibot","OpenLinkProfiler","Perplexity-User","PerplexityBot","PetalBot","Primalbot","python-requests","Re-Animator","Riddler","SafeDNSBot","SBIntuitionsBot","Scrapy","SearchAtlas","SemanticScholarBot","SemrushBot","SentiBot","Sogou","Spider-Pig","sqlmap","StorygizeBot","Test Certificate Info","TikTokSpider","Timpibot","Turnitin","uipbot","vBSEO"]}
{"domain":"attheraces.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["bingbot","*"]}
{"domain":"videy.co","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"wyndhamhotels.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Bingbot","*","Google-HotelAdsVerifier/2.0","Google-HotelAdsVerifier","AdIdxBot"]}
{"domain":"haberturk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"klook.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["Yeti","Baiduspider","Baiduspider-image","Baiduspider-video","Baiduspider-news","Baiduspider-favo","Baiduspider-cpro","Baiduspider-ads","Sogou web spider","Sogou inst spider","Sogou spider","Sogou wap spider","360Spider","360Spider-Image","360Spider-Video","PetalBot","*","AdsBot-Google"]}
{"domain":"kion.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","Googlebot","*"]}
{"domain":"adme.media","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot","Yandex","Mediapartners-Google"]}
{"domain":"aetna.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.az","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"kucoin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bluecava.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"met.no","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oricon.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["Googlebot-News","*","Yahoo-MMCrawler","psbot","BaiduImagespider","BaiduMobaider","TAGGY","Mail.RU_Bot","grapeshot"]}
{"domain":"colorado.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"logitechg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oregon.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","dcbs-google","thunderstonesa","Googlebot","bingbot"]}
{"domain":"google.tt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"besoccer.com","fetch_ok":false,"usable":false,"reason":"other","status":406,"error":null,"attempts":[406,406]}
{"domain":"iq.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tn.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"admixer.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tacobell.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"justpaste.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ottgoods.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"udag.de","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","TimeoutError"]}
{"domain":"eukdns.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"ono.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mcldns.co.uk","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"appdynamics.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"app.google","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":["ENOTFOUND",400]}
{"domain":"axs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Twitterbot"]}
{"domain":"freep.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":287,"stanza_tokens":["anthropic-ai","AmazonBot","Claude","Claude-Web","claudeweb","Amzn-SearchBot","Amzn-User","AndiBot","cohere-ai","ClaudeBot","ChatGPT-User","Googlebot-News","GPTBot","CCBot","Google-Extended","DataForSeoBot","Diffbot","Diffbot-user","Kagibot","FacebookBot","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Scrapy","TurnitinBot","Applebot-Extended","Meta-ExternalAgent","meta-externalagent","OAI-SearchBot","Bytespider","Claude-User","Claude-SearchBot","AI2Bot","AhrefsBot","Ai2Bot-Dolma","Applebot","Arquivo-web-crawler","AspiegelBot","AwarioRssBot","AwarioSmartBot","Barkrowler","coccocbot-web","cohere-training-data-crawler","crawler4j","Crawlspace","dotbot","DuckAssistBot","FriendlyCrawler","Google-CloudVertexBot","https://hada.news","https://www.imediaethics.org","iaskspider/2.0","ICC-Crawler","icc-crawler","ICC Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","LivelapBot","MauiBot","Meltwater","Meta-ExternalFetcher","MJ12bot","Mojeek","MojeekBot","MoodleBot","NeticleBot","Netvibes","PanguBot","PetalBot","SeekportBot","Seekr","SemrushBot","SeznamBot","Sidetrade Indexer Bot","Timpibot","Turnitin","VelenPublicWebCrawler","Webzio-Extended","yacy","yacybot","Yandex","YandexAdditional","YandexAdditionalBot","YandexBot","YandexImages","YandexNews","Baiduspider","Baiduspider-image","BaiduSpider-render","Baiduspider-video","Sogou Web Spider","yisouspider"]}
{"domain":"programmaticx.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AdsBot-Google-Mobile","AdsBot-Google","PetalBot","dotbot","AhrefsBot"]}
{"domain":"loggly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"imo.im","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"r2.dev","fetch_ok":false,"usable":false,"reason":"other","status":500,"error":null,"attempts":["ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE",500,500]}
{"domain":"gencat.cat","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adac.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"byjus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"claro.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Googlebot","Googlebot-Image","Mediapartners-Google","Adsbot-Google"]}
{"domain":"queniurc.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"williamhill.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mcldns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"liftdsp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"haproxy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"bbcgoodfood.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":10,"stanza_tokens":["*","AI2Bot-DeepResearchEval","AI2Bot","AI2Bot-Dolma","Bytespider","CloudflareBrowserRenderingCrawler","CCBot","meta-externalagent","GPTBot","Scope3/2.0"]}
{"domain":"auone.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"bezeqint.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"gpstrackerforpets.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"globalpay.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"nairaland.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"lavozdegalicia.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":110,"stanza_tokens":["*","008 Disallow: /","AddThis.com Disallow: /","admantx Disallow: /","AhrefsBot Disallow: /","BDCbot Disallow: /","Bender Disallow: /","BIXOCRAWLER Disallow: /","bl.uk_lddc_bot Disallow: /","BLEXBot Disallow: /","BUbiNG Disallow: /","Cliqzbot Disallow: /","CNCDialer Disallow: /","crawler4j Disallow: /","CrystalSemanticsBot Disallow: /","CyberAlert Disallow: /","DigExt Disallow: /","discobot Disallow: /","discoverybot Disallow: /","dloader Disallow: /","dloader(NaverRobot) Disallow: /","DOC Disallow: /","dotbot Disallow: /","Download Ninja Disallow: /","DTS Agent Disallow: /","Exabot Disallow: /","Ezooms Disallow: /","FairShare Disallow: /","Fetch Disallow: /","Flamingo_SearchEngine Disallow: /","Genieo Disallow: /","Gigabot Disallow: /","grub-client Disallow: /","Heritrix Disallow: /","heritrix/3.3.0 Disallow: /","HTTrack Disallow: /","ia_archiver Disallow: /","integromedb Disallow: /","IstellaBot Disallow: /","JikeSpider Disallow: /","Jyxobot Disallow: /","k2spider Disallow: /","Kimengi Disallow: /","Kimengi/nineconnections.com Disallow: /","larbin Disallow: /","LexxeBot/1.0 Disallow: /","libwww Disallow: /","linko Disallow: /","Livelapbot Disallow: /","magpie-crawler Disallow: /","Maxthon Disallow: /","MetaURI Disallow: /","Microsoft.URL.Control Disallow: /","MJ12bot Disallow: /","Moreover Disallow: /","Moreoverbot Disallow: /","MSIECrawler Disallow: /","nabot Disallow: /","NaverBot Disallow: /","NerdByNature.Bot Disallow: /","netEstate NE Crawler Disallow: /","NetSeer crawler Disallow: /","Newscan Disallow: /","NextGenSearchBot Disallow: /","NPBot Disallow: /","nutch Disallow: /","Offline Explorer Disallow: /","omgilibot Disallow: /","Orthogaffe Disallow: /","PetalBot Disallow: /","PiplBot Disallow: /","Pixray-Seeker Disallow: /","proximic Disallow: /","psbot Disallow: /","QuerySeekerSpider Disallow: /","rogerbot Disallow: /","seokicks Disallow: /","SEOkicks-Robot Disallow: /","SiteBot Disallow: /","SiteBot/0.1 Disallow: /","sitecheck.internetseer.com Disallow: /","SiteSnagger Disallow: /","Slurp Disallow: /","sogou Disallow: /","Sosospider Disallow: /","spbot Disallow: /","Spinn3r Disallow: /","Teleport Disallow: /","TeleportPro Disallow: /","trendictionbot Disallow: /","trovitBot Disallow: /","TurnitinBot Disallow: /","UbiCrawler Disallow: /","uMBot-LN Disallow: /","UnisterBot Disallow: /","UniversalFeedParser Disallow: /","WBSearchBot Disallow: /","WebCopier Disallow: /","WebReaper Disallow: /","WebStripper Disallow: /"]}
{"domain":"wn.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"whoop.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cib.com.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"irecommend.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"prf.hn","fetch_ok":false,"usable":false,"reason":"other","status":204,"error":null,"attempts":[204]}
{"domain":"seondnsresolve.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"nyaa.si","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot","Baiduspider","Yandex","*"]}
{"domain":"linkvertise.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"taylorfrancis.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":6,"stanza_tokens":["*","GPTBot","ia_archiver","Googlebot","Applebot","adidxbot"]}
{"domain":"uzh.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","msnbot"]}
{"domain":"yodobashi.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cdnhwcohm19.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"goibibo.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"taboolanews.com","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":[503,503,503]}
{"domain":"xfree.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tf1info.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User","anthropic-ai"],"open":[],"path_restricted":["Claude-SearchBot"]},{"family":"perplexity","blocked":["Perplexity-User"],"open":[],"path_restricted":["PerplexityBot"]}],"stanza_token_count":79,"stanza_tokens":["*","Twitterbot","facebookexternalhit","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","Amazonbot","AmazonBuyForMe","anthropic-ai","Applebot-Extended","bigsur.ai","Bytespider","CCBot","ChatGLM","ChatGLM-Spider","ChatGPT-Operator","ChatGPT-User","ClaudeBot","Claude-User","Claude-Web","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","DataForSeoBot","Datenbank Crawler","Devin","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","gemini-deep-research","GoogleAgent-Mariner","Google-Extended","Google-NotebookLM","GPTBot","ICC-Crawler","ImagesiftBot","imageSpider","img2dataset","Kangaroo Bot","KlaviyoAIBot","laion-huggingface-processor","LCC","LinerBot","Manus-User","meta-externalagent","meta-externalfetcher","MistralAI-User","netEstate","netEstate Imprint Crawler","NovaAct","omgili","OmigiliBot","PanguBot","Perplexity-User","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","SemrushBot-SWA","sider.ai","Spider","TavilyBot","TheKnowledgeAI","TimpiBot","TwinAgent","VelenPublicWebCrawler","webzio-extended","YouBot","AhrefsBot","SearchmetricsBot","rogerbot","deepcrawl","OnCrawl","Screaming Frog SEO Spider","MJ12bot","Sistrix","barkrowler"]}
{"domain":"digitaltarget.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"hostatom.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ledauphine.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":10,"stanza_tokens":["*","Mediapartners-Google","Amazonbot","anthropic-ai","Bytespider","CCBot","ChatGPT-User","Claude-Web","Google-Extended","GPTBot"]}
{"domain":"ugent.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"intermedia.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Browsershots","AdsBot-Google"]}
{"domain":"lazada.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"fas.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gva.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mouser.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"orange.pl","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ECONNRESET","TimeoutError"]}
{"domain":"boltdns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"wpenginepowered.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"gupy.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","ClaudeBot","PerplexityBot","Bytespider","Applebot-Extended"]}
{"domain":"tiny.cc","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fourseasons.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"internet-measurement.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"adobecces.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"myangular.life","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"currys.co.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"google.com.gh","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"sme.sk","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot","anthropic-ai"],"open":["Claude-User"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":32,"stanza_tokens":["*","Googlebot","GPTBot","ClaudeBot","CCBot","Bytespider","Google-Extended","meta-externalagent","Amazonbot","Applebot-Extended","PerplexityBot","ChatGPT-User","OAI-SearchBot","anthropic-ai","Claude-Web","cohere-ai","Diffbot","FacebookBot","PetalBot","Timpibot","img2dataset","omgili","Scrapy","DeepSeekBot","chatgpt-operator","claude-searchbot","GrokBot","MistralAI-User","TikTokSpider","AI2Bot","PanguBot","Amzn-SearchBot"]}
{"domain":"tpbid.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"kleinezeitung.at","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unixtools.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"study.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","AhrefsBot","SemrushBot","CCBot","meta-externalagent"]}
{"domain":"mvideo.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"ahnlab.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"theintercept.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":["OAI-SearchBot"],"path_restricted":[]},{"family":"anthropic","blocked":["anthropic-ai"],"open":["ClaudeBot","Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":9,"stanza_tokens":["Amazonbot","anthropic-ai","AhrefsBot","PerplexityBot","CCBot","Google-Extended","GPTBot","ChatGPT-User","*"]}
{"domain":"swissinfo.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hubspotemail.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"turbify.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Googlebot","Googlebot-Mobile","Twitterbot","facebookexternalhit","SiteAuditBot"]}
{"domain":"resetdigital.co","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"imostatic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"redgifs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"eonet.ne.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"atomex.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"gate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"wired.co.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"thenationalnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot-Image"]}
{"domain":"yektanet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-20.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"eater.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":["GPTBot"],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":105,"stanza_tokens":["Amazonbot","Applebot","GPTBot","A6-Indexer","AhrefsBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot-Extended","Baiduspider","betaBot","BLEXBot","botify","brands-bot-logo","Brightbot","Bytespider","calculon","CCBot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","collection@infegy.com","Cotoyogi","Crawl4AI","CrunchBot","deepcrawl","DeepSeekBot","Diffbot","DirBuster","DTS","DuckAssistBot","EasouSpider","FacebookBot","FirecrawlAgent","FriendlyCrawler","GlutenCrawler","Google-CloudVertexBot","Google-Extended","GrapeshotCrawler","iaskspider","ICC-Crawler","ImagesiftBot","img2dataset","JackCrawl","Jetslide","linkapedia_bot","Mediatoolkitbot","MegaIndex","meta-externalagent","meta-externalfetcher","Microsoft BITS","Microsoft URL Control","Microsoft-WebDAV-MiniRedir","MistralAI-User","news-please","Novellum","NTENTbot","Nutch","omgili","Omgilibot","OpenLinkProfiler","Perplexity-User","PerplexityBot","PetalBot","Primalbot","python-requests","Re-Animator","Riddler","SafeDNSBot","SBIntuitionsBot","Scrapy","SearchAtlas","SemanticScholarBot","SemrushBot","SentiBot","Sogou","Spider-Pig","sqlmap","StorygizeBot","Test Certificate Info","TikTokSpider","Timpibot","Turnitin","uipbot","vBSEO","VegeBot","VelenPublicWebCrawler","Web Link Validator","WebCruiser"]}
{"domain":"demon.co.uk","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"EAI_AGAIN","attempts":["EAI_AGAIN","EAI_AGAIN"]}
{"domain":"remotewd.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tranche.finance","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"artnet.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"radio-canada.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"threebit.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"awsapprunner.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fastly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dianping.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":7,"stanza_tokens":["GPTBot","ByteSpider","ToutiaoSpider","koubeispider","koubei.com","amap.com","*"]}
{"domain":"50union.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"beget.pro","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"vivozap.com.br","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"barchart.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"elstc.co","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ricoh.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"4pda.to","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bedbathandbeyond.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"google.com.bh","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"google.mk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"health.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"pch.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sabah.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"newatlas.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AmazonAdBot","Googlebot","*"]}
{"domain":"royalroad.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":11,"stanza_tokens":["*","grapeshot","Getintent Crawler (https://getintent.com/bot.html)","GPTBot","CCBot","Google-Extended","Applebot-Extended","Amazonbot","Bytespider","ClaudeBot","meta-externalagent"]}
{"domain":"bigbasket.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AdsBot-Google","Googlebot-Image"]}
{"domain":"codeforces.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":56,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","MJ12bot","Mediapartners-Google","IsraBot","Orthogaffe","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","fast","grub-client","k2spider","NPBot","WebReaper","Omgilibot","ChatGPT-User","openai-crawler","anthropic-ai","Claude-Web","PerplexityBot","YouBot","Applebot","facebookexternalhit","MetaAI-Bot","HuggingFaceBot","Scrapy","python-requests","curl","Wget"]}
{"domain":"raiplay.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":26,"stanza_tokens":["*","Amazonbot","anthropic-ai","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","FacebookBot","Google-Extended","GPTBot","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Scrapy","TurnitinBot"]}
{"domain":"goldmansachs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","GPTBot","ChatGPT-User"]}
{"domain":"whiterabbitpress.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tv2.dk","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":97,"stanza_tokens":["*","AddSearchBot","AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","Andibot","anthropic-ai","Applebot","Applebot-Extended","Awario","bedrockbot","bigsur.ai","Brightbot 1.0","Bytespider","CCBot","ChatGPT Agent","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Datenbank Crawler","DeepSeekBot","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","Google-Firebase","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","LinerBot","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","meta-webindexer","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NovaAct","OAI-SearchBot","omgili","omgilibot","OpenAI","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","ShapBot","Sidetrade indexer bot","TerraCotta","Thinkbot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","wpbot","YaK","YandexAdditional","YandexAdditionalBot","YouBot"]}
{"domain":"mimecast.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"bhg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"gocache.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mlit.go.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"sportsapi.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"pro-market.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sfchronicle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","anthropic-ai"],"open":[],"path_restricted":["Claude-SearchBot"]}],"stanza_token_count":24,"stanza_tokens":["ai2bot-dolma","anthropic-ai","applebot-extended","bytespider","ccbot","claude-user","claude-web","claudebot","cohere-ai","diffbot","duckassistbot","friendlycrawler","omgili","omgilibot","perplexity-user","perplexitybot","scrapy","timpibot","youbot","webzio-extended","kangaroo-bot","google-extended","AmazonAdBot","*"]}
{"domain":"codeium.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ifixit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":17,"stanza_tokens":["*","Google-Extended","AwarioRssBot","AwarioSmartBot","Baiduspider","DataForSeoBot","Diffbot","ImagesiftBot","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","Scrapy","TurnitinBot"]}
{"domain":"bidtheatre.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"kgns1.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"clamav.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"thisisdax.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID",403]}
{"domain":"clicktale.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"gfk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"chefkoch.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":75,"stanza_tokens":["*","Googlebot","AddSearchBot","AI2Bot","anthropic-ai","Applebot-Extended","BedrockBot","bigsur.ai","Brightbot 1.0","Bytespider","CCBot","ChatGPT Agent","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","Cohere-Training-Data-Crawler","Cotoyogi","Datenbank Crawler","DeepSeek","DeepSeek-Crawler","DeepSeekBot","Devin","Diffbot","DuckAssistBot","EchoboxBot","EtaoSpider","FacebookBot","FireCrawlAgent","GoogleAgent-Mariner","GoogleAgent-Shopping","GPTBot","ICC-Crawler","ImagesiftBot","img2dataset","Kangaroo Bot","Kraken","LinerBot","Linkfluence Yak Bot","Meta-ExternalAgent","Meta-ExternalFetcher","MistralAI-Crawler","MyCentralAIScraperBot","netEstate Imprint Crawler","news-please","NovaAct","ObjectsSearch","omgili","omgilibot","Operator","PanguBot","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","Quora-Bot","quillbot.com","ResearchBot","SBIntuitionsBot","scrapy","SeekrBot","Sidetrade indexer bot","TaraGroup Intelligent Bot","ThinkBot","thinkers-bot","Timpibot","Webzio-Extended","wpbot","YouBot","facebookexternalhit","AmazonAdBot","NanoInteractive","Opebot-v (https://www.1plusx.com)"]}
{"domain":"getgrass.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ghanaweb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","PetalBot","*"]}
{"domain":"amplify.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"cinemark.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pixels.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"st.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"mynet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"exblog.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.sn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"adage.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":15,"stanza_tokens":["*","LinkedInBot","Twitterbot","facebookexternalhit","EchoboxBot","ClaudeBot","CCBot","Google-CloudVertexBot","Google-Extended","Meta-ExternalAgent","OAI-SearchBot","ChatGPT-User","GPTBot","PerplexityBot","Perplexity-User"]}
{"domain":"mixi.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Mediapartners-Google","Bingbot"]}
{"domain":"cribl.cloud","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"wmo.int","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tal.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"forexfactory.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"csiro.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"device9.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"fragrantica.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"csis.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Bytespider"]}
{"domain":"nrich.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wwd.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":26,"stanza_tokens":["*","anthropic-ai","cohere-ai","omgili","omgilibot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ChatGPT","DeepSeek","DeepSeekBot","GPTBot","OAI-SearchBot","ClaudeBot","Claude-Web","Diffbot","FacebookBot","Google-Extended","Meta-ExternalAgent","Meta-ExternalFetcher","PerplexityBot","Timpibot","Webzio-Extended","YouBot","cXensebot"]}
{"domain":"goooooooooooooooooooooooooooooooooooooooooooooooooooooooooogle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tomtom.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","PerplexityBot","AndiBot","ClaudeBot","Claude-User","Claude-SearchBot"]}
{"domain":"willhaben.at","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Mediapartners-Google","yahoo-slurp","msnbot","grapeshot"]}
{"domain":"netvigator.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-03.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"l-tike.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"oath.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"solitaired.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"finviz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"tsn.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nghttp2.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"qh-lb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"wistia.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mossturbo.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"digicert.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"stockx.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iloveimg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"telstra.com.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":32,"stanza_tokens":["Alexa","Amazon-Kendra","AmazonBot","ByteDance","ByteSpider","CC-Crawler","CCBot","ChatGLM","ClearScopeBot","CommonCrawl","DataForSEOBot","Diffbot","Falcon","FlyRiver","Goose","Grammarly","Iaskspider","imagesiftBot","Img2dataset","PetalBot","Quillbot","Scrapy","Sidetrade","Sonic","TikTokSpider","Timpibot","VelenPublicWebCrawler","Swiftbot","Elastic-Crawler Elastic Cloud","Elastic-Crawler","Elastic Cloud","*"]}
{"domain":"liquipedia.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":34,"stanza_tokens":["AdsBot-Google","AdsBot-Google-Mobile","Mediapartners-Google","AdIdxBot","HTTrack","wget","XoviBot","AhrefsBot","AhrefsSiteAudit","Riddler","FDM","ExtLinksBot","SemrushBot","SemrushBot-SA","MJ12bot","QuetextBot","BLEXBot","grapeshot","BluechipBacklinks","barkrowler","GPTBot","ClaudeBot","Claude-Web","cohere-ai","DataForSeoBot","PetalBot","meta-externalagent","AwarioRssBot","AwarioSmartBot","Amazonbot","ChatGPT-User","IbouBot","YouBot","*"]}
{"domain":"iom.int","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"huaweicloud.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iata.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"obsproject.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","CrazyWebCrawler-Spider"]}
{"domain":"contentful.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bandsintown.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"odnoklassniki.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"funnyjunk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":22,"stanza_tokens":["*","TTD-Content","TinEye","\"TinEye\"","CCBot","ChatGPT-User","GPTBot","Google-Extended","anthropic-ai","ClaudeBot","Omgilibot","Omgili","Diffbot","Bytespider","ImagesiftBot","cohere-ai","PerplexityBot","FacebookBot","Amazonbot","YouBot","Applebot-Extended","meta-externalagent"]}
{"domain":"sandiegouniontribune.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":68,"stanza_tokens":["*","AI2Bot","Ai2Bot-Dolma","Amazonbot","Amzn-SearchBot","anthropic-ai","Applebot","Applebot-Extended","Archive.org_bot","Baiduspider","Brightbot 1.0","Bytespider","carbon-umbrella-bot","CCBot","ChatGLM-Spider","ChatGPT-User","ClaudeBot","Claude-Searchbot","Claude-User","Claude-Web","Cohere-training-data-crawler","Cotoyogi","Dalvik","Datenbank Crawler","Diffbot","DocSpotBot","DuckAssistBot","FacebookBot","GnowitNewsbot","Google-Extended","GPTBot","grok-crawler","ia_archiver-web.archive.org","ICC-Crawler","imageSpider","Kangaroo Bot","laion-huggingface-processor","LCC","Mantisbot","Meta-ExternalAgent","MistralAI-User","netEstate Imprint Crawler","OAI-SearchBot","omgili","PanguBot","PerplexityBot","Perplexity-User","PetalBot","SBIntuitionsBot","SirdataBot","Spider","Thinkbot","Timpibot","VelenPublicWebCrawler","Vivaldi","Webzio-extended","YandexBot","YouBot","AmazonAdBot","facebookexternalhit","grapeshot","Mozilla/5.0 (compatible; EchoboxBot/1.0; hash/w4mwnpbXf3MFAbxOkJRw; +http://www.echobox.com)","Twitterbot","Mozilla/5.0 (compatible; NewsRoom.BI/0.1; +http://www.newsroom.bi/bot.html)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36(KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; mrfCompass-Marshall/1.0)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Jukebox/1.0)"]}
{"domain":"fpjs.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"gksmdns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"tpnet.pl","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"ui-dns.de","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dior.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bseindia.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"kabum.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","ia_archiver","Googlebot-Image"]}
{"domain":"cra.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sex-studentki.live","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ns.il","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-10.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"poetryfoundation.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"telegram.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":287,"stanza_tokens":["anthropic-ai","AmazonBot","Claude","Claude-Web","claudeweb","Amzn-SearchBot","Amzn-User","AndiBot","cohere-ai","ClaudeBot","ChatGPT-User","Googlebot-News","GPTBot","CCBot","Google-Extended","DataForSeoBot","Diffbot","Diffbot-user","Kagibot","FacebookBot","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Scrapy","TurnitinBot","Applebot-Extended","Meta-ExternalAgent","meta-externalagent","OAI-SearchBot","Bytespider","Claude-User","Claude-SearchBot","AI2Bot","AhrefsBot","Ai2Bot-Dolma","Applebot","Arquivo-web-crawler","AspiegelBot","AwarioRssBot","AwarioSmartBot","Barkrowler","coccocbot-web","cohere-training-data-crawler","crawler4j","Crawlspace","dotbot","DuckAssistBot","FriendlyCrawler","Google-CloudVertexBot","https://hada.news","https://www.imediaethics.org","iaskspider/2.0","ICC-Crawler","icc-crawler","ICC Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","LivelapBot","MauiBot","Meltwater","Meta-ExternalFetcher","MJ12bot","Mojeek","MojeekBot","MoodleBot","NeticleBot","Netvibes","PanguBot","PetalBot","SeekportBot","Seekr","SemrushBot","SeznamBot","Sidetrade Indexer Bot","Timpibot","Turnitin","VelenPublicWebCrawler","Webzio-Extended","yacy","yacybot","Yandex","YandexAdditional","YandexAdditionalBot","YandexBot","YandexImages","YandexNews","Baiduspider","Baiduspider-image","BaiduSpider-render","Baiduspider-video","Sogou Web Spider","yisouspider"]}
{"domain":"sanook.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot-News","*"]}
{"domain":"sport1.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lgtviot.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gsa.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dexscreener.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"parismatch.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":55,"stanza_tokens":["*","Mediapartners-Google","Meltwater","archive-fr.com","360Spider","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","Baiduspider-image","Baiduspider-video","Brightbot 1.0","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Crawlspace","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","PetalBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","Sogou Spider","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YisouSpider","YouBot"]}
{"domain":"laredoute.fr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"stream-io-api.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"techsmith.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"utk.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"liveramp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rp.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"haproxy.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"drogasil.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"uchile.cl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"google.rs","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"ucar.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wolterskluwer.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"360totalsecurity.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"queniubh.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"bytetcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"evo-games.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"jpn.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"metropoles.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"spanishdict.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":6,"stanza_tokens":["*","CriteoBot/0.1","GPTBot","CCBot","Applebot-Extended","AmazonAdBot"]}
{"domain":"notebooklm.google","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"stylevana.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":11,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","Googlebot-image"]}
{"domain":"womenshealthmag.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AmazonAdBot","Google-InspectionTool","ChatGPT-User","GPTBot","OAI-SearchBot","*"]}
{"domain":"webmotors.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["Google-Extended","GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Adsbot-Google","PetalBot","YandexBot","Baiduspider","Sogou","SEOkicks-Robot","MegaIndex.ru","BLEXBot","*"]}
{"domain":"yamaha.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ajc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":29,"stanza_tokens":["*","Twitterbot","bingbot","AhrefsbBot","SemrushBot","BLEXBot","MJ12bot","AmazonAdBot","Pinterestbot","trendictionbot","NTENTbot","DotBot","rogerbot","Exabot","GetIntent Crawler","Yandex","YandexMobileBot","PhantomJS","SputnikBot","SeznamBot","Cliqzbot","Mail.RU_Bot","Baiduspider","Baiduspider-video","Baiduspider-image","SentiBot","MauiBot (crawler.feedback+wc@gmail.com)","Mappy","WotBot"]}
{"domain":"bongacams.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ashemaletube.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AhrefsBot","PetalBot"]}
{"domain":"mumsnet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","AdsBot-Google\\","APIs-Google","AdsBot-Google-Mobile","Mediapartners-Google"]}
{"domain":"accompany.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"doaj.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"adnkronos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","bingbot"]}
{"domain":"startpage.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rebrand.ly","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"usgovcloud.microsoft","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"jooble.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","GoogleBot","GoogleBot-Mobile","Mediapartners-Google","Yandex","eSobiSubscriber","Exabot","MJ12bot","SemrushBot","SemrushBot-SA"]}
{"domain":"vinted.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":7,"stanza_tokens":["*","grapeshot","GPTBot","ClaudeBot","Google-Extended","Amazonbot","CCBot"]}
{"domain":"ilgiornale.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"emol.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot-News"]}
{"domain":"rapidgator.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"liveworksheets.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"uni-koeln.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","ia_archiver","Nuclei","GPTBot"]}
{"domain":"twitchcdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"academy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"scirp.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"telecinco.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","FacebookBot","facebookexternalhit","Twitterbot/1.0","msnbot","yaanibot"]}
{"domain":"theporndude.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.lt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"homedepot.ca","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"lululemon.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"aemet.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"episerver.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"awsdns-57.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"rtvslo.si","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":21,"stanza_tokens":["*","magpie-crawler","CCBot","omgili","omgilibot","Claude-Web","ClaudeBot","anthropic-ai","cohere-ai","Bytespider","PetalBot","Scrapy","Applebot-Extended","GPTBot","ChatGPT-User","PerplexityBot","Perplexity-User","Google-CloudVertexBot","OAI-SearchBot","YandexAdditional","YandexAdditionalBot"]}
{"domain":"europepmc.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"leprogres.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":10,"stanza_tokens":["*","Mediapartners-Google","Amazonbot","anthropic-ai","Bytespider","CCBot","ChatGPT-User","Claude-Web","Google-Extended","GPTBot"]}
{"domain":"unodc.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"phosagro.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"costco.ca","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"midea.com.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"gemini.google","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wb.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bunnings.com.au","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"foodandwine.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"cigna.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vinted.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":7,"stanza_tokens":["*","grapeshot","GPTBot","ClaudeBot","Google-Extended","Amazonbot","CCBot"]}
{"domain":"beehiiv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","OAI-SearchBot"]}
{"domain":"webnode.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["psbot","*"]}
{"domain":"astro.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_VERIFY_LEAF_SIGNATURE","attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE","UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"de-tencentclb.cloud","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"svr-algorix.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"thanhnien.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"letelegramme.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":248,"stanza_tokens":["Adsbot-Google","AdsBot-Google-Mobile","Applebot","Bingbot","DuckDuckBot","flipboard","Googlebot","Googlebot-Image","Googlebot-Mobile","Googlebot-Video","Mediapartners-Google","Slurp","Taboolabot","Twitterbot","upday","Googlebot-News","Google-InspectionTool","anthropic-ai","Applebot-Extended","ChatGPT-User","Claude-Web","ClaudeBot","DuckAssistBot","Google-Extended","GPTBot","Grok-DeepSearch","GrokBot","cohere-ai","FacebookBot","MistralAI-User","OAI-SearchBot","PerplexityBot","xAI-Grok","5emeRue","5erue","adequat","adequat-systems","AI2Bot","Alexibot","AlvinetSpider","Amazonbot","AmiSoftware","Antenne Hatena","ApocalXExplorerBot","Argus","Ask n read","asknread.com","asterias","Augure","auramundi","BizInformation","Bloomberg","BlowFish/1.0","BotALot","BuiltBotTough","Bullseye/1.0","BunnySlippers","Bytespider","CCBot","Cegbfeieh","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","Cision","coexel","ConveraCrawler","CopyRightCheck","Corporama","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","cydralspider","Diffbot","Digimind","DISCo Pump 3.1","DittoSpyder","dotbot","Download Ninja","downloadexpress","ellisphere","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","eureka","Europresse","Explore","ExtractorPro","Fasterfox","Fetch","Flamingo_SearchEngine","Foobot","gammaSpider","grub-client","hloader","httplib","HTTrack","HTTrack 3.0","humanlinks"]}
{"domain":"niceincontact.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","GPTBot","OAI-SearchBot","Claude-SearchBot","Claude-User","ChatGPT-User","PerplexityBot"]}
{"domain":"work.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Linguee","*","ClaudeBot"]}
{"domain":"bvfcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mijia.tech","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rateyourmusic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":62,"stanza_tokens":["Mediapartners-Google*","Googlebot-Image","googlebot","Twitterbot","msnbot","Slurp","AliyunSecBot","Amazonbot","anthropic-ai","Applebot-Extended","archive.org_bot","AudigentAdBot","AwarioRssBot","AwarioSmartBot","BLEXBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","DuckAssistBot","EchoboxBot","FacebookBot","FirecrawlAgent","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","GPTBot","ImagesiftBot","Jetslide","magpie-crawler","Meta-ExternalAgent","meta-externalagent","Meta-ExternalFetcher","meta-externalfetcher","MyCentralAIScraperBot","NewsNow","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Perplexity-User","Poseidon Research Crawler","quillbot.com","Quora-Bot","Scrapy","SeekrBot","SeznamHomepageCrawler","TaraGroup Intelligent Bot","Timpibot","TurnitinBot","ViennaTinyBot","YouBot","*"]}
{"domain":"easports.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ghostery.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"interno.gov.it","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"xhwebsite.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"narvar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","* Disallow: /thank-you-return-benchmarks-report","* Disallow: /email-subscription-preferences","* Disallow: /ty","* Disallow: /ty/*","* Disallow: /profile-terms-of-service","* Disallow: /profile-privacy-policy"]}
{"domain":"author.today","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["YandexBot","*"]}
{"domain":"microad.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"liputan6.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"beebom.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"decathlon.fr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sf.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","AmazonAdBot","*"]}
{"domain":"dan.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"instabug.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"youradchoices.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":30,"stanza_tokens":["AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","DuckAssistBot","FacebookBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","img2dataset","Meta-ExternalAgent","MyCentralAIScraperBot","omgili","omgilibot","Quora-Bot","TikTokSpider","YouBot","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","*"]}
{"domain":"imedia.cz","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"autohome.com.cn","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"thuvienphapluat.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"dafiti.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","Googlebot","Googlebot-image","Google-Extended","ChatGPT-User","PerplexityBot","CCBot"]}
{"domain":"scalibur.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"tate.org.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rit.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":21,"stanza_tokens":["SemrushBot","SemanticScholarBot","PetalBot","Baiduspider","Yandex","trendictionbot","WinHTTP Example/1.0","ClaudeBot","anthropic-ai","CCBot","ChatGPT-User","GPTBot","*","RIT Storm Crawler","Googlebot","google","SiteimproveBot","SiteimproveBot-Crawler","AHC/1.0","AHC/2.0","AHC/2.1"]}
{"domain":"ondigitalocean.app","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fws.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.co.uz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"sling.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cifraclub.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"plista.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNREFUSED"]}
{"domain":"twnic.net.tw","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",404]}
{"domain":"backlinko.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"mdr.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":29,"stanza_tokens":["applebot-extended","bytespider","ccbot","claudebot","google-extended","cohere-training-data-crawler","deepseekbot","deepseek","deepseek-crawler","diffbot","facebookbot","meta-externalagent","gptbot","grokbot","img2dataset","laiondownloader","mistralai-crawler","omgili","omgilibot","pangubot","Teleport","Webwhacker","Webzip","Net Attache","SiteSnagger","HTTrack","WebCapture","WebSauger","*"]}
{"domain":"elfinanciero.com.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Slurp","PetalBot","grapeshot"]}
{"domain":"fanpage.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"meta.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["Amazonbot","Applebot-Extended","Brightbot","ClaudeBot","Google-Extended","GPTBot","PerplexityBot","PetalBot","Scrapy","uptimerobot","viberbot","YaK","Yandex","Yeti","Applebot","Bingbot","Discordbot","DuckDuckBot","facebookexternalhit","Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News","Google-InspectionTool","LinkedInBot","msnbot","Pinterestbot","Screaming Frog SEO Spider","seznambot","Slurp","teoma","TelegramBot","Twitterbot","*"]}
{"domain":"csfd.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":6,"stanza_tokens":["ClaudeBot","ImagesiftBot","SenutoBot","CCBot","AudigentAdBot","*"]}
{"domain":"dhnet.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":125,"stanza_tokens":["*","Mediapartners-Google","5emeRue","5erue","adequat","adequat-systems","AmiSoftware","Argus","Ask n read","asknread.com","Augure","auramundi","Bloodhound","CCBot","ChatGPT-User","Cision","Claudebot","Cliqzbot","coexel","ConveraCrawler","Corporama","cydralspider","Digimind","Dotbot","Download Ninja","downloadexpress","EDD","ellisphere","eureka","Europresse","Explore","Factiva","Fasterfox","Fetch","FirstRain","gammaSpider","GnowitNewsbot","Google-Extended","GPTBot","grub-client","ia_archiver","ia_archiver-web.archive.org","indexer","infoseek","Jetbot","k2spider","Kantar","kbcrawl","Knowings","larbin","leadbox","libwww","linkfluence","linko","manageo","MaxPointCrawler","mediacompil","Meltawer","Meltwater","mention","Moreover","MSIECrawler","mytwip","newscan-online","NewsNow","Newzbin","NPBot","Nutch","ObjectsSearch","omgili","omgilibot","opinion-tracker","Pimptrain","proxem","QuepasaCreep","Qwam content intelligence","Raven","readability.com","Riddler","rogerbot","scoop.it","score3","SearchmetricsBot","Sindup","sitecheck.internetseer.com","SiteSnagger","spotter","Synthesio","Talkwater","Teleport","TeleportPro","trendeo","trendybuzz","TunitinBot","TurnitinBot","up2news","vecteurplus","Verif","verticalsearch","vsw"]}
{"domain":"razersynapse.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"gov.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"fbs1-t-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"procyclingstats.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"awsdns-51.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"europapress.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":17,"stanza_tokens":["*","Yandex","Mail.Ru","Freshbot","IstellaBot","AhrefsBot","trendictionbot","sentibot","grapeshot","heritrix","MJ12bot","um-IC","SiteAuditBot","Slurp","dotbot","SeekportBot","barkrowler"]}
{"domain":"anota.ai","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"gsu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"docs.rs","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cpsc.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hackernoon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":44,"stanza_tokens":["*","Googlebot","Bingbot","YandexBot","DuckDuckBot","Qwantify","Yeti","Sogou","MojeekBot","ChatGPT-User","ChatGPT-User/2.0","OAI-SearchBot","PerplexityBot","Claude-Web","Claude-SearchBot","Claude-User","DuckAssistBot","Amzn-SearchBot","ManusBot","facebookexternalhit","FacebookBot","Twitterbot","LinkedInBot","Slackbot","Applebot","GPTBot","Google-Extended","GoogleOther","Google-CloudVertexBot","anthropic-ai","ClaudeBot","CCBot","Meta-ExternalAgent","Amazonbot","Applebot-Extended","cohere-ai","Bytespider","Diffbot","AI2Bot","MistralAI-User","Omgilibot","webzio-extended","YouBot","TerracottaBot"]}
{"domain":"vinted.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":7,"stanza_tokens":["*","grapeshot","GPTBot","ClaudeBot","Google-Extended","Amazonbot","CCBot"]}
{"domain":"reserved.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Meta-ExternalAds","*"]}
{"domain":"natalie.mu","fetch_ok":false,"usable":false,"reason":"other","status":405,"error":null,"attempts":[405,405]}
{"domain":"awsdns-23.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"22.cn","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"fon.bet","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"incometax.gov.in","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"illinois.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unctad.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"onefootball.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"msdmanuals.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot-Image","Baiduspider"]}
{"domain":"trustedshops.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":22,"stanza_tokens":["*","MJ12bot","infometrics-bot","AI2Bot","Ai2Bot-Dolma","Bytespider","CCBot","Diffbot","FriendlyCrawler","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","omgili","omgilibot","Scrapy","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended"]}
{"domain":"spdns.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nseindia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"freecodecamp.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"globalso.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["MJ12bot","AhrefsBot","Applebot","SeznamBot","DotBot","SemrushBot"]}
{"domain":"scholastic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gbnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["adsbot-google","*"]}
{"domain":"statefarm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["PetalBot","*"]}
{"domain":"2dehands.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gazeta.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"uni-frankfurt.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":21,"stanza_tokens":["*","MegaIndex.ru","Barkrowler","aranhabot","SEOkicks","DotBot","BLEXBot","MJ12bot","DataForSeoBot","FemtosearchBot","serpstatbot","WebPageTest.org","crawler_faq","GeedoBot","crawler_eb_germany","MetaJobBot","CCBot","VelenPublicWebCrawler","hgfAlphaXCrawl","FullStoryBot","NE Crawler"]}
{"domain":"smartrecruiters.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"o2.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":7,"stanza_tokens":["*","GPTBot","CCBot","PerplexityBot","Perplexity-User","OAI-SearchBot","ChatGPT-User"]}
{"domain":"very.co.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"shareasale.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ura.news","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["Yandex","Mail.Ru","Rambler","Googlebot","AdsBot-Google","Googlebot-Image","Google-Read-Aloud","FeedFetcher-Google","GoogleDocs","GoogleOther","*"]}
{"domain":"qatarairways.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"powerreviews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thomsonreuters.co.uk","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"observer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"playerok.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"hs-sites.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["HubSpot Crawler","*"]}
{"domain":"sage.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","BUbiNG","MJ12bot"]}
{"domain":"51job.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"hu-berlin.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"like-video.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"n11.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ipxserve.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bitchute.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"betterhelp.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"bundestag.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tn.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"awsdns-35.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"go-vip.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"taxtelecom.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"opera-mini.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","TimeoutError"]}
{"domain":"sgvps.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"o3.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"35.in-addr.arpa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"httpwg.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"awsdns-47.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"brightside.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot","Yandex","Mediapartners-Google"]}
{"domain":"lgdacom.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"turkishairlines.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"voodoo-tech.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-42.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"funda.nl","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"xhbox.today","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"onepeloton.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.mn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"yoast.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"luma.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["Googlebot"]}
{"domain":"literotica.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":9,"stanza_tokens":["anthropic-ai","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","GPTBot","Google-Extended","PerplexityBot","*"]}
{"domain":"google.lv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"dnevnik.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","*","BLEXBot"]}
{"domain":"927589452.space","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"war.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.hn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"eb.mil.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uu.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","SiteimproveBot","SiteimproveBot-Crawler"]}
{"domain":"jornada.com.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":11,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","grapeshot"]}
{"domain":"unece.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vivawebhost.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"lidl.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"appsheet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-40.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"robin-flow.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rae.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"is.fi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":46,"stanza_tokens":["*","ChatGPT-user","CCBot","GPTBot","Google-Extended","ImagesiftBot","FacebookBot","Omgilibot","Amazonbot","anthropic-ai","Claude-Web","cohere-ai","omgili","PerplexityBot","YouBot","Bytespider","Diffbot","Applebot-Extended","OAI-SearchBot","ClaudeBot","Google-CloudVertexBot","Meta-ExternalAgent","Meta-ExternalFetcher","Quora-Bot","DataForSeoBot","Webzio-Extended","FriendlyCrawler","ChatGPT-User","meta-externalagent","meta-externalfetcher","TeleportPro","Qwam content intelligence","Timpibot","Kangaroo Bot","AI2Bot","VelenPublicWebCrawler","img2dataset","Sidetrade indexer bot","PanguBot","ISSCyberRiskCrawler","Ai2Bot-Dolma","Claude-SearchBot","Claude-User","MistralAI-User","DeepSeekBot","Perplexity-User"]}
{"domain":"hobbylobby.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","CazoodleBot","dotbot","Gigabot"]}
{"domain":"fsrv.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bank.in","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"freewebs.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"hpsmart.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gmw.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"schema.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"infoworld.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":37,"stanza_tokens":["*","Ai2Bot-Dolma","AI2Bot","anthropic-ai","Applebot-Extended","Arquivo-web-crawler","AwarioRssBot","AwarioSmartBot","Barkrowler","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","crawler4j","dotbot","Diffbot","FacebookBot","FriendlyCrawler","GPTBot","https://hada.news","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","MJ12bot","PanguBot","Scrapy","Seekr","Timpibot","VelenPublicWebCrawler","Webzio-Extended"]}
{"domain":"adultfriendfinder.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.com.na","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"megapath.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"joongang.co.kr","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"chime.aws","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"devline.tv","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"vesselfinder.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Mediapartners-Google","AmazonAdBot","Baiduspider","coccoc","grapeshot","*"]}
{"domain":"utraff.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":22,"stanza_tokens":["*","YandexAccessibilityBot","YandexCalendar","YandexDialogs","YandexDirect","YandexDirectDyn","YandexFavicons","YaDirectFetcher","YandexMobileBot","YandexMarket","YandexMetrika","YandexMobileScreenShotBot","YandexOntoDBAPI","YandexPartner","YandexRCA","YandexRenderResourcesBot","YandexSearchShop","YandexTracker","YandexUserproxy","YandexVideoParser","YandexScreenshotBot","YandexAdditionalBot"]}
{"domain":"pacbell.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"sportsdirect.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"de-tencentclb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"atrustdns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gamoshi.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"consumerfinance.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mercadopago.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Googlebot-Image","Mediapartners-Google","Adsbot-Google","Twitterbot"]}
{"domain":"racingpost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"shorttv.live","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","YisouSpider"]}
{"domain":"ibiblio.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","ia_archiver","Googlebot-Image","Googlebot-News","Mediapartners-Google","psbot"]}
{"domain":"gnupg.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"netcologne.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bambuser.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"images-amazon.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"publico.pt","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"wlvpn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jqueryui.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"demandware.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"zoho.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot-Image","zs-crawler","zoho-scrapy"]}
{"domain":"agenteimmobiliare.info","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sex.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"nobroker.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"chess-results.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","*","AAAA"]}
{"domain":"1und1.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","GPTBot","ChatGPT-User","OAI-SearchBot","Google-Extended","ClaudeBot","Claude-SearchBot","PerplexityBot","SlySearch","SmartDownload","SpankBot","spanner","SuperBot","SupperHTTP","Surfbot","ia_archiver"]}
{"domain":"wyze.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":["TimeoutError",429,429]}
{"domain":"ipvanish.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","deepcrawl https://deepcrawl.com/bot"]}
{"domain":"seloger.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"js7k.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"violity.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"autotrader.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["Vast_Bot","YandexBot","Tag Inspector Spider","*","AdsBot-Google","GPTBot","OAI-SearchBot","ChatGPT-User","Amazonbot","CCBot","Claude-User","Claude-SearchBot","DuckAssistBot","PerplexityBot","Perplexity-User","GrokBot","AhrefsBot","Google-InspectionTool"]}
{"domain":"akamaixbc.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tuasaude.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kidshealth.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Ultraseek","Lucidworks-Anda/2.0/0.8; +;","Lucidworks-Anda/2.0/0.10; +;","*","googlebot"]}
{"domain":"madeiramadeira.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["Alexabot","*","Googlebot","AdsBot-Google","Mediapartners-Google","APIs-Google","GPTBot","PerplexityBot","ClaudeBot"]}
{"domain":"cnnindonesia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot","*"]}
{"domain":"britishmuseum.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","PowerMapper"]}
{"domain":"b92.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AhrefsBot","Bytespider","*"]}
{"domain":"avira.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sportschau.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":21,"stanza_tokens":["*","applebot-extended","bytespider","ccbot","claudebot","google-extended","cohere-training-data-crawler","deepseekbot","deepseek","deepseek-crawler","diffbot","facebookbot","meta-externalagent","gptbot","grokbot","img2dataset","laiondownloader","mistralai-crawler","omgili","omgilibot","pangubot"]}
{"domain":"unm.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dellin.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","YandexBot","Googlebot","AdsBot-Google","AdsBot-Google-Mobile"]}
{"domain":"cosme.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Baidu","Yeti","Yandex","Applebot"]}
{"domain":"lidl.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cscdns.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"journaldesfemmes.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","Mediapartners-Google*","Pinterestbot","dotbot","AhrefsBot","DataForSeoBot","SemrushBot"]}
{"domain":"airbridge.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","PerplexityBot","anthropic-ai","CCBot","Bytespider"]}
{"domain":"tui.co.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ntv.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["TelegramBot","Yandex","*","trendkite-akashic-crawler","PetalBot"]}
{"domain":"parkingcrew.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"transformativeworks.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"volc-dns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"futurism.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"inep.gov.br","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNRESET"]}
{"domain":"awsdns-52.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.com.bo","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"adminml.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"asurahosting.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"royaleapi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["yandex","bing","yahoo","PetalBot","CriteoBot/0.1","bytespider","byte spider","bytedance","OAI-SearchBot","Applebot","ClaudeBot","GPTBot","Amazonbot","ChatGPT-User","CCBot","PerplexityBot","AmazonAdBot","*"]}
{"domain":"fizzopic.org","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"poder360.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"freeconvert.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"vsassets.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"haveibeenpwned.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hostgator.mx","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"military.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","MJ12Bot","Megaindex","AmazonAdBot"]}
{"domain":"lojasrenner.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":30,"stanza_tokens":["OAI – SearchBot","OAI-SearchBot","ChatGPT-User","ChatGPT-User/2.0","GPTBot","anthropic-ai","ClaudeBot","claude-web","Claude-User","PerplexityBot","Perplexity-User","Google-Extended","Gemini-Deep-Research","BingBot","Amazonbot","Applebot","Applebot-Extended","FacebookBot","meta-externalagent","LinkedInBot","Bytespider","DuckAssistBot","cohere-ai","AI2Bot","CCBot","Diffbot","omgili","TimpiBot","YouBot","*"]}
{"domain":"405kk.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"amp-endpoint1.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"aws-thinq-prd.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mendeley.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["AdsBot-Google-Mobile","AdsBot-Google","*","Googlebot-Image"]}
{"domain":"nodes.ac","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"xn--1-wxfc3gwbi.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"thscore.mobi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"em.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AmazonAdBot","*","Googlebot-News"]}
{"domain":"d-zone.ca","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"freeprivacypolicy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":93,"stanza_tokens":["AhrefsBot","AhrefsSiteAudit","YepBot","SemrushBot","SemrushBot-SA","Rogerbot","Exabot","barkrowler","MJ12bot","rogerbot","Dotbot","dotbot","DataForSeoBot","ia_archiver","magpie-crawler","MBCrawler","iubenda-radar","Gigabot","BlackWidow","ChinaClaw","Custo","DISCo","Download\\ Demon","eCatch","EirGrabber","EmailSiphon","EmailWolf","Express\\ WebPictures","ExtractorPro","EyeNetIE","FlashGet","GetRight","GetWeb!","Go!Zilla","Go-Ahead-Got-It","GrabNet","Grafula","HMView","HTTrack","Image\\ Stripper","Image\\ Sucker","Indy\\ Library","InterGET","Internet\\ Ninja","JetCar","JOC\\ Web\\ Spider","larbin","LeechFTP","Mass\\ Downloader","MIDown\\ tool","Mister\\ PiX","Navroad","NearSite","NetAnts","NetSpider","Net\\ Vampire","NetZIP","Octopus","Offline\\ Explorer","Offline\\ Navigator","PageGrabber","Papa\\ Foto","pavuk","pcBrowser","RealDownload","ReGet","SiteSnagger","SmartDownload","SuperBot","SuperHTTP","Surfbot","tAkeOut","Teleport\\ Pro","VoidEYE","Web\\ Image\\ Collector","Web\\ Sucker","WebAuto","WebCopier","WebFetch","WebGo\\ IS","WebLeacher","WebReaper","WebSauger","Website\\ eXtractor","Website\\ Quester","WebStripper","WebWhacker","WebZIP","Wget","Widow","WWWOFFLE","Xaldon\\ WebSpider","Zeus"]}
{"domain":"aylanetworks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AdsBot-Google-Mobile","AdsBot-Google","PetalBot","dotbot","AhrefsBot"]}
{"domain":"heute.at","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adelaide.edu.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"toptal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tf1.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User","anthropic-ai"],"open":[],"path_restricted":["Claude-SearchBot"]},{"family":"perplexity","blocked":["Perplexity-User"],"open":[],"path_restricted":["PerplexityBot"]}],"stanza_token_count":79,"stanza_tokens":["*","Twitterbot","facebookexternalhit","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","Amazonbot","AmazonBuyForMe","anthropic-ai","Applebot-Extended","bigsur.ai","Bytespider","CCBot","ChatGLM","ChatGLM-Spider","ChatGPT-Operator","ChatGPT-User","ClaudeBot","Claude-User","Claude-Web","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","DataForSeoBot","Datenbank Crawler","Devin","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","gemini-deep-research","GoogleAgent-Mariner","Google-Extended","Google-NotebookLM","GPTBot","ICC-Crawler","ImagesiftBot","imageSpider","img2dataset","Kangaroo Bot","KlaviyoAIBot","laion-huggingface-processor","LCC","LinerBot","Manus-User","meta-externalagent","meta-externalfetcher","MistralAI-User","netEstate","netEstate Imprint Crawler","NovaAct","omgili","OmigiliBot","PanguBot","Perplexity-User","PhindBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","SemrushBot-SWA","sider.ai","Spider","TavilyBot","TheKnowledgeAI","TimpiBot","TwinAgent","VelenPublicWebCrawler","webzio-extended","YouBot","AhrefsBot","SearchmetricsBot","rogerbot","deepcrawl","OnCrawl","Screaming Frog SEO Spider","MJ12bot","Sistrix","barkrowler"]}
{"domain":"repec.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_PACKET_LENGTH_TOO_LONG","attempts":["ERR_SSL_PACKET_LENGTH_TOO_LONG","ERR_SSL_PACKET_LENGTH_TOO_LONG"]}
{"domain":"zol.com.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"slt.lk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tcdnlive.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"bystys.tech","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bdstatic.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"sponichi.co.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"clouddatahub.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"manchestereveningnews.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":[],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":17,"stanza_tokens":["*","semetrical","GPTBot","OAI-SearchBot","Applebot-Extended","PerplexityBot","Perplexity-ai","CCBot","YouBot","ClaudeBot","anthropic-ai","Claude-Web","grapeshot","bingbot","AmazonAdBot","ozone","Meta-ExternalAgent"]}
{"domain":"imganalytics.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"worldpopulationreview.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"androidauthority.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"usvisa-info.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"irctc.co.in","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ETIMEDOUT"]}
{"domain":"hextom.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"maze.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"superbthemes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","facebookexternalhit","facebookexternalhit/1.1","facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)","meta-externalagent","meta-externalagent/1.1","meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)","Googlebot"]}
{"domain":"dawn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"faberlic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ctrls.in","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"auckland.ac.nz","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eloqua.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENETUNREACH"]}
{"domain":"activtrak.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-12.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"winbuzz.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"cloudhub.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"haplat.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"rhinternal.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"jeuxvideo.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"adserv.ai","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"qunaer.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","TimeoutError"]}
{"domain":"brightcovecdn.com","fetch_ok":false,"usable":false,"reason":"other","status":500,"error":null,"attempts":["ENOTFOUND",500,500]}
{"domain":"bill.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"newsmax.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"dpreview.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":21,"stanza_tokens":["*","facebookexternalhit","Googlebot","bingbot","msnbot","AhrefsBot","baiduspider","sogou","dotbot","GoogleOther","magpie-crawler","weborama-fetcher","ias_crawler","360Spider","GPTBot","CCBot","anthropic-ai","Amazonbot","Claude-Web","ClaudeBot","PetalBot"]}
{"domain":"zepto.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"askfilo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wpzoom.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dana.id","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"diamond.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"getpublica.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"bajajfinserv.in","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"midilibre.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":240,"stanza_tokens":["*","5emeRue","5erue","adequat","adequat-systems","Alexibot","AlvinetSpider","Amazonbot","AmiSoftware","Antenne Hatena","anthropic-ai","ApocalXExplorerBot","Argus","Ask n read","asknread.com","asterias","Augure","auramundi","BackDoorBot/1.0","BizInformation","Black Hole","Bloodhound","Bloomberg","BlowFish/1.0","BotALot","BraveBot","BuiltBotTough","Bullseye/1.0","BunnySlippers","Bytespider","CCBot","Cegbfeieh","ChatGPT-User","OAI-SearchBot","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","Cision","Claude-Web","ClaudeBot","coexel","cohere-ai","ConveraCrawler","CopyRightCheck","Corporama","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","cydralspider","Diffbot","Digimind","DISCo Pump 3.1","DittoSpyder","dotbot","Download Ninja","downloadexpress","EDD","ellisphere","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","eureka","Europresse","Explore","ExtractorPro","FacebookBot","Factiva","Fasterfox","Fetch","Flamingo_SearchEngine","flipboard","Foobot","gammaSpider","Google-Extended","GPTBot","grub-client","Harvest/1.5","hloader","httplib","HTTrack","HTTrack 3.0","humanlinks","ia_archiver","ia_archiver-web.archive.org","Igentia","indexer","InfoNaviRobot","infoseek","JennyBot","Jetbot","JikeSpider","k2spider","Kantar","kbcrawl","Kenjin Spider","Knowings","larbin","leadbox"]}
{"domain":"threatdown.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sunat.gob.pe","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"grigorik.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"eaton.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"sanofi-synthelabo.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"thesimsresource.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"flingo.tv","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ECONNRESET"]}
{"domain":"upravel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"relevant-digital.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sniffies.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kurir.rs","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fanatics.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["bingbot","Pinterest","AhrefsBot","AspiegelBot","CCbot","*"]}
{"domain":"iaai.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"truoptik.com","fetch_ok":false,"usable":false,"reason":"other","status":521,"error":null,"attempts":[521,"TimeoutError",521]}
{"domain":"careers360.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Mediapartners-Google"]}
{"domain":"housing.com","fetch_ok":false,"usable":false,"reason":"other","status":406,"error":null,"attempts":[406,406]}
{"domain":"comparethemarket.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":11,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","ia_archiver"]}
{"domain":"carrefour.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":27,"stanza_tokens":["*","Googlebot","Googlebot-News","Googlebot-Image","Googlebot-Video","Googlebot-Mobile","TwengaBot","Pinterestbot","Google-CloudVertexBot","asterias","BackDoorBot/1.0","Black Hole","BlowFish/1.0","BotALot","BuiltBotTough","Bullseye/1.0","BunnySlippers","Cegbfeieh","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","CopyRightCheck","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","DittoSpyder"]}
{"domain":"usu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"youm7.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","Googlebot","Googlebot-Mobile","Mediapartners-Google","Googlebot-Image","Googlebot-News","Google-Extended","GPTBot","PerplexityBot","Amazonbot","meta-externalagent","APIs-Google","CCBot","ClaudeBot","OAI-SearchBot"]}
{"domain":"prothomalo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"onedio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["yaanibot","grapeshot","*","OAI-SearchBot","GPTBot"]}
{"domain":"kundelik.kz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","*","BLEXBot"]}
{"domain":"tekion.xyz","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"withbuddies.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bouncex.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"akc.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"playdigo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ipcheckerapi.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"awsdns-41.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ambito.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"journaldemontreal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User"]}],"stanza_token_count":149,"stanza_tokens":["Googlebot","Googlebot-Image","Googlebot-Mobile","Slurp","MSNBot","YandexNews","YandexBot","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","larbin","libwww","ZyBORG","Download Ninja","wget","grubclient","k2spider","NPBot","WebReaper","TurnitinBot","EmailCollector","EmailWolf","CopyRightCheck","Black Hole","Titan","NetMechanic","CherryPicker","EmailSiphon","WebBandit","Crescent","NICErsPRO","ProWebWalker","CheeseBot","ia_archiver","ia_archiver/1.6","MIIxpc","Telesoft","Website Quester","moget/2.1","WebZip/4.0","Mister PiX","WebSauger","NetAnts","WebAuto","TheNomad","WWWCollectorE","RMA","libWeb/clsHTTP","asterias","httplib","turingos","spanner","InfoNaviRobot","Harvest/1.5","Bullseye/1.0","Mozilla/4.0 (compatible; BullsEye; Windows 95)","Crescent Internet ToolPak HTTP OLE Control v.1.0","CherryPickerSE/1.0","CherryPickerElite/1.0","WebBandit/3.50","DittoSpyder","SpankBot","BotALot","lwptrivial/1.34","lwptrivial","Wget/1.6","BunnySlippers","URLy Warning","Wget/1.5.3","LinkWalker","cosmos","moget","hloader","humanlinks","LinkextractorPro","Mata Hari","LexiBot","Web Image Collector","The Intraformant","True_Robot/1.0","True_Robot","BlowFish/1.0","JennyBot","MIIxpc/4.2","BuiltBotTough","ProPowerBot/2.14","BackDoorBot/1.0","toCrawl/UrlDispatcher","WebEnhancer"]}
{"domain":"cafebazaar.ir","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lsapp.eu","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",404]}
{"domain":"euro.com.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["*","AdsBot-Google","AdsBot-Google-Mobile","Mediapartners-Google","Googlebot","Googlebot-Image","GPTBot","ChatGPT-User","Claude-Web","PerplexityBot","GoogleOther","OAI-SearchBot","ClaudeBot"]}
{"domain":"tally.so","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"securedvisit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gaana.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"altavista.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":47,"stanza_tokens":["*","ADmantX","AlphaBot","Amazonbot","anthropic-ai","AwarioRssBot","AwarioSmartBot","BLEXBot","Brightbot","Buzzbot","Bytespider","CCBot","ChatGPT-User","claritybot","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","huggingface","ImagesiftBot","img2dataset","magpie-crawler","Meltwater","Neevabot","news-please","NewsNow","Nutch","OAI-SearchBot","omgili","omgilibot","http://panscient.com","Perplexity-ai","PerplexityBot","PetalBot","PiplBot","http://scoop.it","Scrapy","Seekr","SentiBot","SeznamBot","TurnitinBot","YouBot","ZumBot"]}
{"domain":"thomsonreuters.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"amtrak.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aptrinsic.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"stihi.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"use.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"quago.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"proximus.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"claro.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","claroBuscador"]}
{"domain":"gucci.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"vonedge.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"elcorreo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":[],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot"],"open":[],"path_restricted":["Claude-User","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":19,"stanza_tokens":["Mediapartners-Google","GoogleBot-Image","Twitterbot","grapeshot","*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Claude-SearchBot","Google-Extended","GPTBot","meta-externalagent","PerplexityBot","OAI-SearchBot","ia_archiver","archive.org_bot","ia_archiver-web.archive.org"]}
{"domain":"mql5.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["Mail.RU_Bot","MJ12bot","ExaBot","CrazyWebCrawler-Spider","SemrushBot","AhrefsBot","BLEXBot","Yandex","*"]}
{"domain":"dandomain.dk","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dnacdn.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"gujarat.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"1filmyfly.fyi","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"13.in-addr.arpa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"lynk.id","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"credly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"datagrail.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rusoska.biz","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"rfa.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"getepic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["msnbot-rss","*"]}
{"domain":"cardekho.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","*","grapeshot"]}
{"domain":"teleport.media","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unrealities.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"isaidub.love","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"bendingspoons.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["ia_archiver","archive.org_bot"]}
{"domain":"heraldo.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":117,"stanza_tokens":["*","008","AddThis.com","admantx","BDCbot","Bender","bl.uk_lddc_bot","BLEXBot","BUbiNG","CNCDialer","CrystalSemanticsBot","CyberAlert","discobot","discoverybot","DOC","DotBot","Download Ninja","Exabot","Ezooms","Fetch","Flamingo_SearchEngine","grub-client","heritrix","HTTrack","integromedb","IstellaBot","JikeSpider","Jyxobot","k2spider","larbin","LexxeBot/1.0","libwww","linko","magpie-crawler","Maxthon","MJ12bot","Moreoverbot","MSIECrawler","NerdByNature.Bot","netEstate NE Crawler","NetSeer crawler","Newscan","NextGenSearchBot","NPBot","Offline Explorer","Pixray-Seeker","psbot","rogerbot","seokicks","SiteBot","SiteBot/0.1","sitecheck.internetseer.com","SiteSnagger","sogou","Sosospider","spbot","Spinn3r","Teleport","TeleportPro","trovitBot","TurnitinBot","UbiCrawler","UnisterBot","WBSearchBot","WebCopier","WebReaper","WebStripper","WebZIP","WeSEE:Search","wget","wotbox","Yasni","Zao","Zealbot","ZyBORG","anthropic-ai","AI2Bot","Ai2Bot-Dolma","AmazonBot","ApifyBot","ApifyWebsiteContentCrawler","Applebot-Extended","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DeepSeekBot","Diffbot","FacebookBot","FirecrawlAgent","GPTBot","iAskBot","iaskspider","ICC-Crawler","Kangaroo Bot"]}
{"domain":"sb.by","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","Googlebot","Googlebot-News","YandexNews"]}
{"domain":"buzzfeednews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":39,"stanza_tokens":["msnbot","*","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot-Extended","CCBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-Web","Claude-User","cohere-ai","Diffbot","discobot","dotbot","FacebookBot","google-extended","Google-Extended","Googlebot","Googlebot-Image","GPTBot","omgilibot","PerplexityBot","TurnitinBot","TtoYouBot","magpie-crawler","Slurp","yacybot","meta-externalagent","Meta-ExternalFetcher","OAI-SearchBot","Bytespider","DuckAssistBot","Perplexity-User","Timpibot","WebGPTBot","Scope3/2.0 (scope3.com)","AmazonAdBot"]}
{"domain":"haisoft.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"dm.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thenounproject.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"findmykids.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","YandexBot"]}
{"domain":"akakce.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"5movierulz.capital","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"seesaa.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"cudasvc.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"cartocdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"vdyel.com","fetch_ok":false,"usable":false,"reason":"other","status":521,"error":null,"attempts":[521,521,"TimeoutError"]}
{"domain":"contaboserver.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"hkjc.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"cnn.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"chinatax.gov.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"mca.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_VERIFY_LEAF_SIGNATURE","attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE","TimeoutError"]}
{"domain":"gambleaware.org","fetch_ok":false,"usable":false,"reason":"other","status":500,"error":null,"attempts":[500,500,"TimeoutError"]}
{"domain":"list-manage1.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"libertaddigital.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"chunichi.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":67,"stanza_tokens":["*","ChatGPT-User","Google-CloudVertexBot","Google-Extended","GPTBot","OAI-SearchBot","AhrefsBot","AI2Bot","Amazonbot","anthropic-ai","Applebot-Extended","Arquivo-web-crawler","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","Claude-Web","ClaudeBot","Claude-SearchBot","Claude-User","coccocbot-web","cohere-ai","cohere-training-data-crawler","Crawlspace","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Gensparkbot","https://hada.news","ICC-Crawler","ImagesiftBot","https://www.imediaethics.org","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","magpie-crawler","Meta-ExternalAgent","Meta-ExternalFetcher","Mojeek","MojeekBot","MoodleBot","Netvibes","NewsNow","omgili","omgilibot","PanguBot","PerplexityBot","Perplexity-ai","Perplexity-User","PetalBot","Scrapy","Seekr","SemrushBot","SeznamBot","Timpibot","Turnitin","TurnitinBot","VelenPublicWebCrawler","Webzio-Extended","yacy","yacybot","Yandex","YandexAdditional","YandexAdditionalBot","YouBot"]}
{"domain":"countryliving.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["AmazonAdBot","Google-InspectionTool","ChatGPT-User","GPTBot","OAI-SearchBot","Google-Extended","*"]}
{"domain":"up.pt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"runnr.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"discovermagazine.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"compass.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"kitco.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"chavesnamao.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":53,"stanza_tokens":["*","GPTBot","ClaudeBot","Google-Extended","Meta-ExternalAgent","Applebot-Extended","DeepSeekBot","Grok","GrokBot","Amazonbot","CCBot","cohere-ai","sitebot","Slurp","CazoodleBot","Cazoodlebot","sistrix","MJ12bot","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","wget","grub-client","k2spider","NPBot","WebReaper","psbot","Exabot","Speedy","dotbot","Bloglines/3.1","Jyxobot/1","cityreview","WijuBot"]}
{"domain":"warcraftlogs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":15,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","AmazonAdBot","Verity/1.1 (https://gumgum.com/verity; verity-support@gumgum.com)","Baiduspider","Baiduspider-render","Yandex"]}
{"domain":"tn.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rtbf.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":20,"stanza_tokens":["*","AI2Bot","Amazonbot","Applebot-Extended","anthropic-ai","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","Diffbot","DuckAssistBot","FacebookBot","Google-Extended","GPTBot","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","PerplexityBot"]}
{"domain":"nme.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":22,"stanza_tokens":["*","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","DataForSeoBot","GPTBot","Google-Extended","magpie-crawler","omgili","omgilibot","PetalBot","PerplexityBot","SemrushBot","Scrapy","TurnitinBot","weborama-fetcher"]}
{"domain":"dodois.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"inkscape.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":4,"stanza_tokens":["*","GPTBot","Google-Extended","Applebot-Extended"]}
{"domain":"pracuj.pl","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"megaphone.fm","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"s-cloud.fi","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"goo-net.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","meta-externalagent"]}
{"domain":"awsdns-50.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"v-tun.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ENOTFOUND"]}
{"domain":"cafepress.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"punchng.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"schwaebische.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":15,"stanza_tokens":["*","Google-Extended","GPTBot","ChatGPT-User","CCBot","ClaudeBot","Bytespider","Amazonbot","Applebot-Extended","ia_archiver","Arquivo-web-crawler","anthropic-ai","OAI-SearchBot","PerplexityBot","Claude-Web"]}
{"domain":"evri.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lis-skins.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","Googlebot","*"]}
{"domain":"awsdns-49.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"repocket.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"citigroup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"playbattlegrounds.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"unh.edu","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ECONNRESET"]}
{"domain":"prreqcroab.icu","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Googlebot","Googlebot-Image","AdsBot-Google","GoogleOther","*"]}
{"domain":"europeana.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"themeansar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"meetcircle-netgear.co","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tripledotapi.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zxcs.nl","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","DEPTH_ZERO_SELF_SIGNED_CERT","DEPTH_ZERO_SELF_SIGNED_CERT"]}
{"domain":"carview.co.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",503]}
{"domain":"blesk.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["*","AhrefsBot","AhrefsSiteAudit","AwarioBot","BLEXBot","Bytespider","CriteoBot","DataForSeoBot","DotBot","GrapeshotCrawler","grapeshot","MJ12bot","MegaIndex","MojeekBot","SemrushBot","YandexBot","Yeti","YisouSpider"]}
{"domain":"ada.support","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"login.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cults3d.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.rw","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"ub.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["PetalBot","*","TurnitinBot"]}
{"domain":"sportinglife.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"biobiochile.cl","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Twitterbot","*","Mediapartners-Google","bingbot","PetalBot","Google-Extended"]}
{"domain":"worldwildlife.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tenten.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"di.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":32,"stanza_tokens":["CCBot","ChatGPT-User","GPTBot","Google-Extended","Google-CloudVertexBot","Omgilibot","Omgili","FacebookBot","ClaudeBot","Claude-SearchBot","Claude-User","Diffbot","DuckAssistBot","PerplexityBot","Perplexity-User","cohere-ai","cohere-training-data-crawler","Meta-ExternalAgent","Meta-ExternalFetcher","Meta-WebIndexer","Timpibot","Webzio-Extended","YouBot","Amazonbot","Bytespider","anthropic-ai","OAI-SearchBot","VelenPublicWebCrawler","Gemini-Deep-Research","ICC-Crawler","MistralAI-User","*"]}
{"domain":"uvic.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"allegro.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dlink.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"successfactors.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Nuclei*","*jndi*","*"]}
{"domain":"crisp.chat","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uitm.edu.my","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"americanas.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":14,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","PerplexityBot","FirecrawlAgent","AndiBot","ExaBot","PhindBot","YouBot","GPTBot","CCBot","Google-Extended","Googlebot","Bingbot"]}
{"domain":"qwant.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"smythstoys.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","OAI-SearchBot","GPTBot","PerplexityBot","CazoodleBot","dotbot","Gigabot"]}
{"domain":"tcd.ie","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"maersk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["AhrefsBot","BernardBot","Baiduspider","*"]}
{"domain":"bigpond.net.au","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"sbercloud.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"awsdns-18.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"etsystatic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nykaa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nameshift.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"almasryalyoum.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"prebid.cloud","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mygaru.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"syr.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"torn.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"travian.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gmx.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mywot.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"scotsman.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":[],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":12,"stanza_tokens":["*","AmazonAdBot","anthropic-ai","Applebot-Extended","GPTBot","Google-Extended","ClaudeBot","Claude-Web","Perplexity-User","PerplexityBot","OAI-SearchBot","Bytespider"]}
{"domain":"carvana.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.by","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"tue.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Fasterfox","Teleport","TeleportPro","wget","BlackWidow"]}
{"domain":"btinternet.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"fussball.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"txxx.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"money.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":8,"stanza_tokens":["*","GPTBot","CCBot","Bytespider","PerplexityBot","Perplexity-User","OAI-SearchBot","ChatGPT-User"]}
{"domain":"eurogirlsescort.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gaijinent.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tencent-cloud.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"douyinstatic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"colossusssp.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"awsdns-48.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"daumcdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"codefusion.technology","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"appdomain.cloud","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bancsabadell.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AhrefsBot","*"]}
{"domain":"cnews.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","grapeshot","Googlebot-News"]}
{"domain":"pinklion.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"hitta.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Facebot","*"]}
{"domain":"noip.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ajax.systems","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pair.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"elperiodico.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":29,"stanza_tokens":["008","UnisterBot","JikeSpider","Zealbot","WebStripper","WebCopier","Fetch","Teleport","HTTrack","magpie-crawler","MJ12bot","DotBot","spbot","BLEXBot","BUbiNG","Yandex","Baiduspider","OAI-SearchBot","ChatGPT-User","GPTBot","ClaudeBot","Meta-ExternalAgent","PerplexityBot","Google-Extended","Googlebot","*","Twitterbot","facebookexternalhit","AmazonAdBot"]}
{"domain":"nationalgeographic.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":["OAI-SearchBot"],"path_restricted":[]},{"family":"anthropic","blocked":["anthropic-ai"],"open":["ClaudeBot","Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":21,"stanza_tokens":["Amazonbot","anthropic-ai","Bytespider","CCBot","ChatGPT-User","Claude-Web","FacebookBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","PerplexityBot","cohere-ai","omgili","omgilibot","Diffbot","ImagesiftBot","Applebot-Extended","dotbot","rogerbot"]}
{"domain":"kinescopecdn.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",404]}
{"domain":"home.ne.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","UND_ERR_CONNECT_TIMEOUT"]}
{"domain":"claudeusercontent.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"vanityfair.it","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"easy4ip.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"nngroup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["AccompanyBot","AwarioRssBot","AwarioSmartBot","AhrefsBot","BLEXBot","BLP_bbot","CCBot","chimebot","MJ12bot","Screaming Frog SEO Spider","Semanticbot","SemrushBot","startmebot","*"]}
{"domain":"adtec.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"workspaceone.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"worldatlas.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AhrefsSiteAudit"]}
{"domain":"jorudan.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AmazonAdBot","Amazonbot"]}
{"domain":"researchnow.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"minecraftskins.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lamoda.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"klevu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot-Image","*"]}
{"domain":"navyfederal.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ome.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pbskids.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":4,"stanza_tokens":["*","Googlebot","GPTBot","CCBot"]}
{"domain":"togetter.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":51,"stanza_tokens":["bingbot","*","Feedly","Twitterbot","Mediapartners-Google","Google-Display-Ads-Bot","AdsBot-Google-Mobile","AdsBot-Google","CriteoBot","Slurp","Megalodon","ia_archiver","AhrefsBot","SemrushBot","MJ12bot","BLEXBot","GPTBot","ChatGPT-User","Applebot-Extended","CCBot","ImagesiftBot","ClaudeBot","claude-web","PerplexityBot","Meta-ExternalAgent","Meta-WebIndexer","DuckAssistBot","DiffBot","cohere-ai","OAI-SearchBot","anthropic-ai","AI2Bot","Riddler","MLBot","JUST-CRAWLER","DuckDuckBot","Sogou web spider","Rogerbot","Omgilibot","Omgili","peer39_crawler","Bytespider","ias-or","Scrapy","Copilot","Google-NotebookLM","Diffbot","PanguBot","MistralAI-User","EtaoSpider","Claude-SearchBot"]}
{"domain":"fmb.la","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"stradivarius.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"anysex.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"homes.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","bingbot","GoogleOther"]}
{"domain":"birkenstock.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot","Googlebot-image"]}
{"domain":"24.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","facebookexternalhit"]}
{"domain":"rd.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"1mg.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"outsideonline.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":52,"stanza_tokens":["Applebot","BingBot","BingPreview","ChatGPT-User","Claude-SearchBot","Claude-User","DuckDuckBot","facebookexternalhit","Grok-DeepSearch","Googlebot","GoogleOther","LinkedInBot","meta-externalfetcher","meta-webindexer","MistralAI-User","OAI-SearchBot","PerplexityBot","Perplexity-User","Slackbot","Twitterbot","xAI-Grok","AI2Bot","AI2Bot-Dolma","AliyunSecBot","Amazonbot","anthropic-ai","Applebot-Extended","bigsur.ai","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","DeepseekBot","Diffbot","FacebookBot","GPTBot","GrokBot","iAskBot","ImagesiftBot","meta-externalagent","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","Petalbot","Timpibot","webzio","webzio-extended","Googlebot-Image","Googlebot-Video","*"]}
{"domain":"joker.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"signupgenius.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)","facebookexternalhit/1.1","facebookcatalog/1.0","Mediapartners-Google","Baiduspider","AmazonAdBot","Google-Adstxt-Crawler","Amazon-Ad-Crawl","IndexExchange","adstxt.trade","Sovrn-crawler","GumGum-Crawler","rubicon-project-crawler","OpenX","PubMatic-AdsTxt-Crawler","Criteo-Ads-Txt-Crawler","FreestarAdQualityBot/1.0","*"]}
{"domain":"allmusic.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"alz.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aniwatchtv.to","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"walkerplus.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"midasbuy.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"thepoizon.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nexters.team","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fravega.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"cntraveler.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","FacebookExternalHit","Twitterbot","Pinterestbot","LinkedInBot","TikTokSpider","Storebot-Google","Google-InspectionTool","AmazonAdBot","Google-Extended","Google-CloudVertexBot","GoogleOther","Applebot-Extended","Amazonbot","meta-webindexer","meta-externalagent","meta-externalfetcher","ClaudeBot","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","cohere-training-data-crawler","cohere-ai","CCBot","PanguBot","PetalBot","Bytespider","Diffbot","DuckAssistBot","MistralAI-User","Timpibot","Webzio","Webzio-Extended","archive.org_bot","ia_archiver","ia_archiver-web.archive.org","heritrix"]}
{"domain":"athletic.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"towersflowerss.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"hvg.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","Googlebot","Bingbot","BingPreview","Pinterestbot","facebookexternalhit","Applebot","OSZKbot","upday","Mediapartners-Google"]}
{"domain":"dzcdn.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Twitterbot"]}
{"domain":"gameforge.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","PerplexityBot","Google-Extended"]}
{"domain":"zbporn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AhrefsBot"]}
{"domain":"d-net.pro","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","ENOTFOUND"]}
{"domain":"lok-lok.cc","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"qiita.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"expo.dev","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kino-teatr.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":17,"stanza_tokens":["Slurp","Mail.Ru","bingbot","msnbot","AhrefsBot","discobot","Ezooms","MJ12bot","Flamingo_SearchEngine","magpie-crawler","Rootle Crawler","pimeyes.com crawler","weborama-fetcher","ias-or","Yandex","YandexImages","*"]}
{"domain":"hna.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":248,"stanza_tokens":["*","Amazonbot","AlibabaBot","Ai2Bot-Dolma","Applebot-Extended","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","Claude-Web","CloudVertexBot","cohere-training-data-crawler","Cotoyogi","Datenbank Crawler","Diffbot","FacebookBot","Google-Extended","GoogleOther","GPTBot","ICC-Crawler","imageSpider","Kangaroo Bot","laion-huggingface-processor","LCC","meta-externalagent","netEstate Imprint Crawler","omgili","PanguBot","PerplexityBot","SBIntuitionsBot","Spider","Timpibot","VelenPublicWebCrawler","webzio-extended","YouBot","008","Dataprovider.com","dcrawl","HelloworkJobPostingBot","HTTrack","HTTrack 3.0","KrawlerBot","MetaInspector","MetaJobBot","newspaper","Nutch","Offline Explorer","OpenindexSpider","Potions","Scrapy","ServerHunterSpider","StatsDroneBot","CloudflareBrowserRenderingCrawler","aa","AhrefsBot","Attracta","Barkrowler","BrightEdge Crawler","BLEXBot","CaliberBot","Caliperbot","ClarityBot","Clearscopebot","Cloudtrellis","cludo.com","Cocolyzebot","cognitiveSEO Crawler","contentking","Convermax","Cxense","DataForSeoBot","DataForSEO Bot","DomainStatsBot","DotBot","Dragonbot","Huckabot","Huckabuy Bot","hypestat","iaskspider","img2dataset","James BOT","LinkCheck by Siteimprove","LinkChecker Bot","linkchecker.pro","linkdexbot","LinksIndexerBot","MarketGoo","MBCrawler","MegaIndex.ru","MJ12bot","Moz dotbot","Moz rogerbot","Nitro-","NitroBot","online-webceo-bot","Prerender","Pro Sitemaps","Readable","RevvimGort","rogerbot-crawler"]}
{"domain":"wikiwand.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"laodong.vn","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"awsdns-05.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bms.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wetest.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":59,"stanza_tokens":["*","Almaden","ASPSeek","Axmo","booch","DTS Agent","Downloader","EmailCollector","EmailSiphon","EmailWolf","Expired Domain Sleuth","Franklin Locator","Gaisbot","grub","HughCrawler","iaea.org","lcabotAccept","IconSurf","Iltrovatore-Setaccio","Indy Library","IUPUI","Kittiecentral","larbin","lwp-trivial","MetaTagRobot","Missigua Locator","NetResearchServer","NextGenSearch","NPbot","Nutch","ObjectsSearch","Oracle Ultra Search","PEERbot","PictureOfInternet","PlantyNet","QuepasaCreep","ScSpider","SOFT411","spider.acont.de","Sqworm","SSM Agent","TAMU","TheUsefulbot","TurnitinBot","Tutorial Crawler","TutorGig","WebCopier","WebZIP","ZipppBot","Wotbox","Wget","mozDex","ZoominfoBot","spyglassbot","SeznamBot","Nimbostratus-Bot","DigExt","Cincraw","BLEXBot"]}
{"domain":"sofi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","archive.org_bot","ia_archiver"]}
{"domain":"usbank.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cnstrc.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"sift.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","GPTBot","ChatGPT-User","PerplexityBot","ClaudeBot","Google-Extended","Amazonbot","Applebot-Extended","CCBot"]}
{"domain":"wordstream.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AhrefsSiteAudit","AhrefsBot","*"]}
{"domain":"uplynk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"hdporncomics.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"videa.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"aptoide.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oecd-ilibrary.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"rust-lang.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"google.com.qa","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"browserleaks.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uninet.net.mx","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","TimeoutError"]}
{"domain":"callofduty.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"mrgcdn.ru","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":["ENOTFOUND",503,503]}
{"domain":"jnbhi.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"coub.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"9animetv.to","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"amplreq.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"t13.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"pavietnam.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"algolianet.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"awsdns-13.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"famousbirthdays.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"namemc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"jnj.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"simplecast.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ensonhaber.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bankier.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":5,"stanza_tokens":["Bard","CCbot","GPTBot","Google-Extended","*"]}
{"domain":"google.kz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"bunshun.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"umontreal.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vivaldi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"182682.xyz","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","attempts":["ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"smartape.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-38.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-39.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"teslamotors.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["EHOSTUNREACH",403]}
{"domain":"fareharbor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"5ch.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"tucows.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"nestle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ntv.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":11,"stanza_tokens":["*","GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","PerplexityBot","TurnitinBot","cohere-ai","OAI-SearchBot","QuillBot"]}
{"domain":"gjacky.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"controlup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","SemrushBot","msnbot","SemrushBot-SA"]}
{"domain":"incapsecuredns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"beforward.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"livekit.cloud","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"cleveland.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","ia_archiver","archive.org_bot","heritrix","My-heritrix-crawler","Timpibot","omgili","Bytespider","DuckAssistBot","Applebot-Extended","Google-Extended","Google-CloudVertexBot","GoogleOther","GPTBot","CCBot","Amazonbot","FacebookBot","cohere-ai","cohere-training-data-crawler","Omgilibot","ClaudeBot","Diffbot","PanguBot","MistralAI-User","webz.io","DeepSeekBot","AmazonAdBot","Amzn-SearchBot","Amzn-User","Claude-SearchBot","Claude-User","PetalBot","Meta-ExternalFetcher","Meta-WebIndexer","OAI-SearchBot","ChatGPT-User","PerplexityBot","Perplexity-User"]}
{"domain":"sklum.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Pinterestbot"]}
{"domain":"futbol24.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"1filmyfly.top","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"amzn.eu","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"sudinfo.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":134,"stanza_tokens":["Mediapartners-Google","Googlebot","Googlebot-Image","Googlebot-Mobile","Googlebot-News","Googlebot-Video","Adsbot-Google","Twitterbot","Applebot","Bingbot","SiteAuditBot","Publication-Access-for-Facebook","facebookexternalhit","Flipboard","FlipboardProxy","upday","AhrefsBot","AhrefsSiteAudit","5emeRue","5erue","adequat","adequat-systems","AmiSoftware","AwarioRssBot","AwarioSmartBot","Argus","Ask n read","asknread.com","Augure","auramundi","Bloodhound","Cision","coexel","ConveraCrawler","Corporama","cydralspider","Digimind","Download Ninja","downloadexpress","EDD","ellisphere","eureka","Europresse","Explore","Factiva","Fasterfox","Fetch","gammaSpider","grub-client","HTTrack","ia_archiver","ia_archiver-web.archive.org","indexer","infoseek","Jetbot","k2spider","Kantar","kbcrawl","Knowings","larbin","leadbox","libwww","linkfluence","linko","manageo","mediacompil","Meltwater","mention","Moreover","MSIECrawler","mytwip","newscan-online","NewsNow","Newzbin","NPBot","ObjectsSearch","Offline Explorer","opinion-tracker","Pimptrain","proxem","QuepasaCreep","Qwam content intelligence","Raven","readability.com","scoop.it","score3","Sindup","sitecheck.internetseer.com","SiteSnagger","spotter","Synthesio","Talkwater","Teleport","TeleportPro","trendeo","trendybuzz","TunitinBot","TurnitinBot","up2news","vecteurplus"]}
{"domain":"baltimoresun.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":68,"stanza_tokens":["*","AI2Bot","Ai2Bot-Dolma","Amazonbot","Amzn-SearchBot","anthropic-ai","Applebot","Applebot-Extended","Archive.org_bot","Baiduspider","Brightbot 1.0","Bytespider","carbon-umbrella-bot","CCBot","ChatGLM-Spider","ChatGPT-User","ClaudeBot","Claude-Searchbot","Claude-User","Claude-Web","Cohere-training-data-crawler","Cotoyogi","Dalvik","Datenbank Crawler","Diffbot","DocSpotBot","DuckAssistBot","FacebookBot","GnowitNewsbot","Google-Extended","GPTBot","grok-crawler","ia_archiver-web.archive.org","ICC-Crawler","imageSpider","Kangaroo Bot","laion-huggingface-processor","LCC","Mantisbot","Meta-ExternalAgent","MistralAI-User","netEstate Imprint Crawler","OAI-SearchBot","omgili","PanguBot","PerplexityBot","Perplexity-User","PetalBot","SBIntuitionsBot","SirdataBot","Spider","Thinkbot","Timpibot","VelenPublicWebCrawler","Vivaldi","Webzio-extended","YandexBot","YouBot","AmazonAdBot","facebookexternalhit","grapeshot","Mozilla/5.0 (compatible; EchoboxBot/1.0; hash/w4mwnpbXf3MFAbxOkJRw; +http://www.echobox.com)","Twitterbot","Mozilla/5.0 (compatible; NewsRoom.BI/0.1; +http://www.newsroom.bi/bot.html)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36(KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; mrfCompass-Marshall/1.0)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Jukebox/1.0)"]}
{"domain":"cloudflare.dev","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"hrzn-nxt.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"telex.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["facebookexternalhit"]}
{"domain":"fortnite.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"esteri.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"punchmedia.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"azuregov-dns.us","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"webpt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lanouvellerepublique.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":52,"stanza_tokens":["DuckDuckBot","Mediapartners-Google","Googlebot","Googlebot-Image","Googlebot-Mobile","GoogleProducer","Googlebot-Video","AdsBot-Google","Googlebot_Nauxeo","Qwantify","Qwant-news","VoilaBot","msnbot","Slurp","bingbot","Twitterbot","facebookexternalhit","Applebot","Bingbot","Facebot","grapeshot","Flipboard","FlipboardProxy","weborama-fetcher","FeedFetcher-Google","ias_crawler","ias-sg","ias-au","ias-jp","ias-ie","ias-or","ias-va","ias-sync","admantx-apacas","admantx-euaspb","admantx-usaspb","admantx-ussy01","admantx-ussy02","admantx-ussy03","admantx-ussy04","admantx-eusy01","admantx-eusyncbatch01","admantx-usbatch","admantx-sap","aday-rediffusion","PanoramaPQR","SeedtagBot","AmazonAdBot","IAB-Tech-Lab","TTD-Content","Googlebot-News","*"]}
{"domain":"aidcgroup.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"google.ge","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"usergate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"epluscdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cnbcindonesia.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tiermaker.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot-Image","*"]}
{"domain":"appcues.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"maxroll.gg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":107,"stanza_tokens":["\"008\"","AddSearchBot","AI2Bot","Ai2Bot-DeepResearchEval","Ai2Bot-Dolma","Amazonbot","AmazonBuyForMe","Amzn-SearchBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot","Applebot-extended","atlassian-bot","AutoRAG","AwarioSmartBot","AzureAI-SearchBot","Big Sur AI","bigsur.ai","Brandwatch","Bravebot","Brightbot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DDM-DCipher*","DDM-DCipher/1.0.7","DDM*","DeepSeekBot","Devin","Diffbot","Direqt Anomura","DuckAssistBot","Echobot Bot","ExaBot","FacebookBot","Factset_spyderbot","FirecrawlAgent","GPTBot","HTTrack","iAsk","iAskBot","iaskspider","ICC Crawler","ICC-Crawler","ImageSiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","Kunato","laion-huggingface-processor","LCC","LINER Bot","LinerBot","LinkupBot","Manus-User","meta-externalagent","meta-externalfetcher","meta-webindexer","mistral.ai","MistralAI-User","netEstate Imprint Crawler","NovaAct","Novellum AI Crawl","Nutch","OAI-SearchBot","Offline Explorer","omgili","opencode","PanguBot","PeopleInc-DCipher-Scraper/1.1.0","Perplexity-User","PerplexityBot","PetalBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","Scrapy","SeekrBot","SemrushBot-OCOB","SemrushBotSwa","ShapBot","Spider","TavilyBot","TerraCotta","Timpibot","Trae","TwinAgent"]}
{"domain":"squarespace-cdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"c212.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"viifwjox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"encyclopedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot-Image"]}
{"domain":"pixlr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"juicyads.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iowa.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fiu.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"playmogo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"myisolved.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"globaltimes.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ulta.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","AdsBot-Google-Mobile","AdsBot-Google","Bytespider","Meta-ExternalAgent","OAI-SearchBot","GPTBot","CCBot","PerplexityBot","ClaudeBot","Google-Extended","Bingbot"]}
{"domain":"datatables.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tvspielfilm.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":15,"stanza_tokens":["*","Amazonbot","Anchor Browser","Bytespider","CCBot","ClaudeBot","FacebookBot","Google-CloudVertexBot","GPTBot","meta-externalagent","Novellum","PetalBot","ProRataInc","Timpibot","FriendlyCrawler"]}
{"domain":"me-qr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"upm.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zalando.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Pinterestbot","*"]}
{"domain":"spicychat.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","SemrushBot"]}
{"domain":"trovaprezzi.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["EtaoSpider","psbot","NPBot-1/2.0","NPBot","Googlebot","AdsBot-Google-Mobile","AdsBot-Google","Bingbot","AdldxBot","*"]}
{"domain":"minkabu.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":2,"stanza_tokens":["*","GPTBot"]}
{"domain":"skyrock.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"tokyo.lg.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bartleby.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"emgsound.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"opentext.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["AwarioBot","Barkrowler","BUbiNG","ContentKing","DotBot","ecoresearchCrawler","MJ12bot","SeznamBot","Vegi bot","Bytedance","Bytespider","CCBot","AhrefsBot","SemrushBot","SemrushBot-BA","*"]}
{"domain":"shiksha.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ixigo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AdIdxBot","MSNBot","Yandex","Baiduspider","Bingbot"]}
{"domain":"awsdns-25.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-58.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"akenai.click","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-07.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"poringa.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mohfw.gov.in","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"ehow.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Mediapartners-Google"]}
{"domain":"gelderlander.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":30,"stanza_tokens":["*","Twitterbot","GPTBot","ChatGPT-User","CCBot","anthropic-ai","Amazonbot","Applebot-Extended","Bytespider","ClaudeBot","Claude-Web","cohere-ai","Diffbot","Google-Extended","Meta-ExternalFetcher","PerplexityBot","YouBot","DeepSeekBot","MistralAI-User","MistralAI-User/1.0","PetalBot","Timpibot","YandexAdditional","YandexAdditionalBot","YandexBot","YandexRenderBot","Perplexity-User","coccocbot-web","OAI-SearchBot","Claude"]}
{"domain":"ssg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["Googlebot","Googlebot-Image","GoogleOther","GoogleOther-Image","Google-InspectionTool","Yeti","Bingbot","Daum","Slurp","Baiduspider","ZumBot","Storebot-Google","Ads-Naver","Twitterbot","*"]}
{"domain":"kathimerini.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":26,"stanza_tokens":["*","Amazonbot","Anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","Cohere-ai","DataForSeoBot","FacebookBot","Google-Extended","GPTBot","ImagesiftBot","Magpie-crawler","meta-externalagent","NewsNow","Omgili","Omgilibot","Peer39_crawler","Peer39_crawler/1.0","PerplexityBot","YouBot"]}
{"domain":"admhmao.ru","fetch_ok":false,"usable":false,"reason":"other","status":447,"error":null,"attempts":[447,447]}
{"domain":"kth.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zvuk.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":418,"error":null,"attempts":[418,418]}
{"domain":"ss2.us","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"omappapi.com","fetch_ok":false,"usable":false,"reason":"other","status":501,"error":null,"attempts":["TypeError",501,501]}
{"domain":"ancestry.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","proximic","sogou spider","Google-Extended"]}
{"domain":"jagranjosh.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Mediapartners-Google","Googlebot-News"]}
{"domain":"main.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"pricecharting.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"inxxx.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*\rDisallow: /player/float.php\rDisallow: /player/html.php\rDisallow: /redirect_cs.php\rDisallow: /*?action=*$\rDisallow: /*mode=async*$\rDisallow: /*function=get_block*$\rDisallow: /404.php\rDisallow: /logout/\rDisallow: /email/\rDisallow: /payments/\rDisallow: /login/"]}
{"domain":"allthefallen.moe","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"hentaiera.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"macromill.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gentoo.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":95,"stanza_tokens":["AhrefsBot","AddSearchBot","AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","Andibot","anthropic-ai","Applebot","Applebot-Extended","Awario","bedrockbot","bigsur.ai","Brightbot 1.0","Bytespider","CCBot","ChatGPT Agent","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Datenbank Crawler","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","FacebookBot","facebookexternalhit","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","LinerBot","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NovaAct","OAI-SearchBot","omgili","omgilibot","OpenAI","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","Thinkbot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","wpbot","YaK","YandexAdditional","YandexAdditionalBot","YouBot","*"]}
{"domain":"catchthemes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rosneft.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"gopro.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"topuniversities.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"brew.sh","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tim.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"prismic.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"joinmastodon.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"futbollibretv.su","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lavoz.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":[],"path_restricted":["GPTBot","OAI-SearchBot"]}],"stanza_token_count":7,"stanza_tokens":["SemrushBot-SA","*","ChatGPT-User","OpenAI","ChatGPT","FacebookBot","Meta AI"]}
{"domain":"boomtrain.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"thegioididong.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","PetalBot"]}
{"domain":"91mobiles.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Googlebot-News","Mediapartners-Google","008","Baiduspider"]}
{"domain":"theaustralian.com.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":68,"stanza_tokens":["ScreamingFrogSEOSpider","Mobian","proximic","Outbrain","Yahoo Ad monitoring","SemrushBot","facebookexternalhit","ias_crawler","identrics","MeltwaterNews","links.streem.com.au","LinkedInBot","redditbot","Mediapartners-Google","Google-InspectionTool","Adsbot-Google","Storebot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","APIs-Google","ChatGPT-User","GPTBot","OAI-SearchBot","Google-Ads-Creatives-Assistant","Google-adwords","Google-amphtml","Google-cloudvertexbot","Google-play-newsstand","Google-read-aloud","Google-Safety","Google-site-verification","Google-Structured-Data-Testing-Tool","Googleimageproxy","Applebot","AppleNewsBot","adidxbot","admantx","admantx-usbatch/3.1","AmazonAdBot","Chrome-Lighthouse","Verity","Leikibot","parselybot","peer39_crawler/1.0","pingdom","Pinterestbot","PubMatic Crawler Bot","Slack-ImgProxy","Slackbot","Slackbot-LinkExpanding","thetradedesk","Uptimebot","yahoo-linkpreview","Googlebot","Googlebot-Image","Googlebot-News","Googlebot-Video","GoogleOther","GoogleOther-Image","GoogleOther-Video","Bingbot","bingpreview","MicrosoftPreview","DuckDuckBot","Slurp","yahoo","Google-Extended","*"]}
{"domain":"sophosupd.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"metunic.com.tr","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"bp.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"awsdns-59.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"usemessages.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"meshok.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Yandex","SemrushBot","SemrushBot-SA","PetalBot","Amazonbot","*"]}
{"domain":"neopets.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","MJ12bot","proximic","PetalBot","Mediapartners-Google","IAB-Tech-Lab","Verity/1.1"]}
{"domain":"thairath.co.th","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","grapeshot","ias_crawler","AmazonAdBot"]}
{"domain":"jbl.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"w.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","ClaudeBot","anthropic-ai","Google-Extended","Applebot-Extended","PerplexityBot","Bytespider","CCBot","Copilot"]}
{"domain":"dnsdomen.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pornoxo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AhrefsBot","PetalBot"]}
{"domain":"seznamzpravy.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","SeznamSocialBot","AI"]}
{"domain":"portfolio.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zhangle.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"ngrok.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"comdirect.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ltx71"]}
{"domain":"zalando.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Pinterestbot","*"]}
{"domain":"haber7.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"plrm.zone","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"awsdns-02.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-11.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-24.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cartacapital.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"drift.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[404,200]}
{"domain":"seriouseats.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"verizonwireless.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","dotbot"]}
{"domain":"footmercato.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pghub.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"manus.im","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","facebook","Twitterbot","Slackbot","LinkedInBot","Whatsapp","TelegramBot","redditbot"]}
{"domain":"delaware.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["archive.org_bot","*"]}
{"domain":"gitbook.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"exponea.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sikayetvar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"instant-gaming.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"piwik.pro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vanguardngr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"mlive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","ia_archiver","archive.org_bot","heritrix","My-heritrix-crawler","Timpibot","omgili","Bytespider","DuckAssistBot","Applebot-Extended","Google-Extended","Google-CloudVertexBot","GoogleOther","GPTBot","CCBot","Amazonbot","FacebookBot","cohere-ai","cohere-training-data-crawler","Omgilibot","ClaudeBot","Diffbot","PanguBot","MistralAI-User","webz.io","DeepSeekBot","AmazonAdBot","Amzn-SearchBot","Amzn-User","Claude-SearchBot","Claude-User","PetalBot","Meta-ExternalFetcher","Meta-WebIndexer","OAI-SearchBot","ChatGPT-User","PerplexityBot","Perplexity-User"]}
{"domain":"awsdns-06.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"protonmail.ch","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"scoop.it","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"fnn.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"xnhau.hot","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"b2c.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zapimoveis.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"fut.gg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"truepeoplesearch.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"treehugger.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"awsdns-45.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fgov.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"windguru.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Yandex","SemrushBot","bingbot","MJ12bot","meta-externalfetcher"]}
{"domain":"epicentrk.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","AdsBot-Google","Googlebot-Image","APIs-Google","AdsBot-Google-Mobile"]}
{"domain":"xsolla.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["atlassian-bot","*"]}
{"domain":"fhaccess.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vwgroup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"huaweicloud-dns.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kariyer.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ou.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"beget.tech","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"rubika.ir","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"vkcs.cloud","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tivoservice.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"yasyadong.tv","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"EAI_AGAIN","attempts":["EAI_AGAIN","EAI_AGAIN"]}
{"domain":"yako03.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"bitcoin.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"maariv.co.il","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ok.xxx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"paramount.tech","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"getty.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["SortSite","*","AhrefsBot","Yeti"]}
{"domain":"boosty.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","Googlebot"]}
{"domain":"mingky03.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"visuy.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"name-servers.gr","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"kyoto-u.ac.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["dotbot","*"]}
{"domain":"pensador.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"comfy.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["bingbot","*"]}
{"domain":"moviesda18.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"my-best.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"morganstanley.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"5movierulz.holiday","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"redcdn.pl","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"kisskh.co","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"windowsphone.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",200]}
{"domain":"rokutime.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"crowd-umlaut.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"biography.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AmazonAdBot","Google-InspectionTool","ChatGPT-User","GPTBot","OAI-SearchBot","*"]}
{"domain":"rferl.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AhrefsBot"]}
{"domain":"camwhores.tv","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"giallozafferano.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":16,"stanza_tokens":["*","Claude-Web","anthropic-ai","cohere-ai","PerplexityBot","seekr","Meltwater","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"taya99aa.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"gmpwaqf.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"kemono.cr","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"azure.cn","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"moe.gov.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"allegro.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["msnbot"]}
{"domain":"ydparty02.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"chimpstatic.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"uba.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"alcmpn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"rqmob.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"crowncoinscasino.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"5movierulz.theater","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-09.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ngrok.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mizuhobank.co.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"pfizer.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"psdops.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"athome.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Sogou web spider","Googlebot","bingbot","MJ12bot","Baiduspider"]}
{"domain":"icims.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-30.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"jup9.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"codedish.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["Baiduspider","Baiduspider-image","Baiduspider-video","Baiduspider-news","Baiduspider-favo","Baiduspider-cpro","Baiduspider-ads","Sogou","360Spider","YisouSpider","YodaoBot","Bytespider","Ezooms","BLEXBot","EtaoSpider","HuaweiBot","QihooSpider","MJ12bot","Sosospider","JikeSpider"]}
{"domain":"xn--pckua2a7gp15o89zb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Baiduspider","BaiduMobaider","BaiduImagespider"]}
{"domain":"virginmedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cafef.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"superuser.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":418,"error":null,"attempts":[418,418]}
{"domain":"twstalker.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"eksisozluk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":17,"stanza_tokens":["*","Mediapartners-Google","Microsoft URL Control","grapeshot","GPTBot","anthropic-ai","CCBot","ChatGPT-User","FacebookBot","Google-Extended","ClaudeBot","Omgilibot","Omgili","Diffbot","Bytespider","ImagesiftBot","cohere-ai"]}
{"domain":"niedersachsen.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vidgf.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"control4.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"daraz.com.bd","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tpgi.com.au","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"greylabeldelivery.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"warframe.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"djangoproject.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"taxtelecom.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sfdc.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bash.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ivanti.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"itemsatis.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"nova.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","Amazonbot","Bytespider","CCBot","Cohere-ai","Omgili","Omgilibot","YouBot"]}
{"domain":"raptive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["anthropic-ai"],"open":["ClaudeBot","Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":9,"stanza_tokens":["*","anthropic-ai","Claude-Web","CCbot","FacebookBot","Google-Extended","GPTBot","PiplBot","Google-InspectionTool"]}
{"domain":"prmutv.co","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"av19.to","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"flexport.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"decolar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","AdsBot-Google","Baiduspider","google-hoteladsverifier"]}
{"domain":"filmyzilla35.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":11,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","Mediapartners-Google"]}
{"domain":"bunkr.cr","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"awsdns-43.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mp4moviez.ad","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"ladesk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mcldns.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"cmtelematics.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"virgilio.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","dotbot","dotbot/1.0","Baiduspider","MegaIndex.ru","AhrefsBot","grapeshot","Acunetix","AwarioRssBot","AwarioSmartBot","BomboraBot","BUbiNG","Yandex","AmazonAdBot"]}
{"domain":"adevarul.ro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vidtrx.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"beeg.porn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"5movierulz.navy","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"lectormangaa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"amarujala.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":28,"stanza_tokens":["CCBot","anthropic-ai","Omgilibot","Omgili","Diffbot","Bytespider","ImagesiftBot","cohere-ai","ClaudeBot","Claude-Web","Meta-ExternalAgent","Meta-ExternalFetcher","Applebot","Applebot-Extended","Amazonbot","Meta-WebIndexer","GPTBot","Google-Extended","FacebookBot","OAI-SearchBot","ChatGPT-User","PerplexityBot","FirecrawlAgent","AndiBot","ExaBot","PhindBot","YouBot","*"]}
{"domain":"astro-seek.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.hr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"vide.la","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"bain.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"xxxbule.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"blogs.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"no-ip.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_VERIFY_LEAF_SIGNATURE","attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE","UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"twitcasting.tv","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"knigavuhe.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"taz.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":4,"stanza_tokens":["Slurp","*","GPTBot","PetalBot"]}
{"domain":"prodoctorov.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AhrefsBot","*"]}
{"domain":"xnhau.fan","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"hal.science","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":50,"stanza_tokens":["*","SemanticScholarBot","TurnitinBot","MegaIndex.ru","SemrushBot","SemrushBot-SA","Seekport Crawler","ltx71 - (http://ltx71.com/)","AhrefsBot","Riddler","CCBot","ChatGPT-User","Diffbot","GPTBot","Google-Extended","Omgilibot","omgili","FacebookBot","cohere-ai","anthropic-ai","Bytespider","Amazonbot","Applebot","Applebot-Extended","PerplexityBot","YouBot","AdsBot-Google","AwarioRssBot","AwarioSmartBot","ClaudeBot","Claude-Web","DataForSeoBot","FriendlyCrawler","GoogleOther","ImagesiftBot","magpie-crawler","Meltwater","peer39_crawler","PiplBot","Seekr","Meta-ExternalAgent","Timpibot","SummalyBot","facebookexternalhit","ICC-Crawler","img2dataset","OAI-SearchBot","PetalBot","Scrapy","VelenPublicWebCrawler"]}
{"domain":"whatismyipaddress.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mofa.go.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"prolific.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rpp.pe","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":38,"stanza_tokens":["GPTBot","CCBot","anthropic-ai","Claudebot","Bytespider","Amazonbot","FacebookBot","cohere-ai","DataForSeoBot","Diffbot","omgili","omgilibot","YouBot","SemrushBot","dotbot","MJ12bot","Rogerbot","Scrapy","TurnitinBot","magpie-crawler","NewsNow","news-please","AwarioRssBot","AwarioSmartBot","PiplBot","Google-Extended","Googlebot","Googlebot-News","Googlebot-Image","Googlebot-Video","Googlebot-Mobile","PerplexityBot","Bingbot","Claude-SearchBot","Claude-User","Petalbot","OAI-SearchBot","*"]}
{"domain":"homeip.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"hqporner.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mediavitrina.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"hostwindsdns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"dbankcdn.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"storm.mg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["meta-externalagent","*"]}
{"domain":"hotnews.ro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"patagonia.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"awsdns-29.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"optusnet.com.au","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"awsdns-33.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tagsrvcs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"24sata.hr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Mediapartners-Google","grapeshot"]}
{"domain":"auerproxy.de","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-16.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"jetblue.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gala.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":128,"stanza_tokens":["AddSearchBot","AhrefsBot","AI2Bot","AihitBot","AllenAI","Amazon-Kendra","Andibot","Anomura","anthropic-ai","Applebot-Extended","archive.org_bot","arquivo-web-crawler","arquivo.pt","Awario","BacklinkCrawler","Baiduspider","Barkrowler","BedrockBot","bigsur.ai","BLEXBot","Bloodhound","Bravebot","Brightbot 1.0","browsertrix","brozzler","Buddybot","BuiltWith","Bytespider","CCBot","ChatGLM-Spider","ChatGPT Agent","Cincraw","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","Cohere-Training-Data-Crawler","Cotoyogi","CrawlSpace","cydralspider","DataForSeoBot","Datenbank Crawler","DeepSeek","DeepSeekBot","Devin","Diffbot","dotbot","downloadexpress","DuckAssistBot","Echobot Bot","EchoboxBot","FactSet_SpyderBot","Fasterfox","FireCrawlAgent","Flamingo_SearchEngine","FriendlyCrawler","gammaSpider","Google-Extended","GoogleAgent-Mariner","GPTBot","ia_archiver","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Jetslide","Kangaroo Bot","Kraken","laion-huggingface-processor","LCC","LinerBot","Linkfluence Yak Bot","LinkWalker","magpie-crawler","MistralAI-User","MistralAI-User/1.0","MJ12bot","msnbot-media/1.1","msnbot-media/2.0b","MyCentralAIScraperBot","netEstate Imprint Crawler","news-please","NewsNow","NovaAct","ObjectsSearch","omgili","omgilibot","Operator","PanguBot","Panscient","panscient.com","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","proximic","QualifiedBot"]}
{"domain":"mordorintelligence.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"8x8.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":[429,429,429]}
{"domain":"osf.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","PingBot","PerplexityBot","GPTBot","BaiduSpider","Meta-ExternalAgent"]}
{"domain":"mathrubhumi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":19,"stanza_tokens":["*","GPTBot","ChatGPT-User","OAI-SearchBot","PerplexityBot","Amazonbot","Bytespider","ClaudeBot","Claude-Web","Meta-ExternalAgent","FacebookBot","CCBot","Anthropic-ai","YouBot","Google-Extended","xAI-Crawler","DeepSeek","facebookexternalhit","Facebot"]}
{"domain":"rule34.us","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"abxxx.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","Baiduspider","AhrefsBot","MJ12bot","008","BLEXBot","sistrix","ia_archiver","proximic","SiteExplorer"]}
{"domain":"cretaforce.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"meitustat.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"aniwaves.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wikimapia.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"sytes.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"goodnotes-devops.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-22.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"maine.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"digi24.ro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["facebookexternalhit","*"]}
{"domain":"lngtd.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"mofcom.gov.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"signalfx.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_VERIFY_LEAF_SIGNATURE","attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE","UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"medicalxpress.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","ClaudeBot","GPTBot","FacebookBot","AhrefsBot","CCbot","Bytespider","cohere-ai","Google-Extended","Applebot-Extended","Scrapy","FriendlyCrawler","ChatGLM-Spider","DataForSeoBot","ImagesiftBot","TapnativeScrapingBot"]}
{"domain":"otempo.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","PetalBot","AmazonAdBot"]}
{"domain":"bigbadtoystore.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"idealista.pt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sbb.rs","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"nami.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"hornbach.de","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"telenor.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot","Googlebot-image","AdsBot-Google","*"]}
{"domain":"nii.ac.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"poliziadistato.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pixieset.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["msnbot","MJ12bot","AhrefsBot","Yandex","bingbot","*"]}
{"domain":"liu.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zalando.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Pinterestbot","*"]}
{"domain":"statnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":25,"stanza_tokens":["*","AI2Bot-Dolma","Amazonbot","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot-Extended","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","Diffbot","FacebookBot","FirecrawlAgent","Google-Extended","GoogleOther","GPTBot","imageSpider","meta-externalagent","omgili","PanguBot","Spider","Timpibot","Webzio-Extended"]}
{"domain":"savana.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":29,"stanza_tokens":["Googlebot","AdsBot-Google-Mobile","AdsBot-Google","Mediapartners-Google","AdsBot-Google-Mobile-Apps","Googlebot-Image","Googlebot-Video","Google-Site-Verification","facebookexternalhit","facebookcatalog","Twitterbot","bingbot","Applebot","Yahoo! Slurp","SemrushBot","SiteAuditBot","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SemrushBot-CT","SplitSignalBot","SemrushBot-COUB","seo-audit-check-bot","rogerbot","dotbot","AhrefsBot","AhrefsSiteAudit","OAI-SearchBot","*"]}
{"domain":"zus.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"voz.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"chinacloudapi.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"northeastern.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kettledroopingcontinuation.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"grserver.gr","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"news.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","GoogleBot","Flipboard","FlipboardProxy","Yandex"]}
{"domain":"dotmetrics.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sdn.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"misspark.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"xv-ru.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ku.dk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"telangana.gov.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rcn.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"equant.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","TimeoutError"]}
{"domain":"a47b.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"awsdns-08.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nvidiagrid.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"microchip.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"manoramaonline.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":6,"stanza_tokens":["AmazonAdBot","*","Mediapartners-Google","GPTBot","CCBots","DeepSeek"]}
{"domain":"theonion.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":34,"stanza_tokens":["*","Amazonbot","anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","FacebookBot","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","GPTBot","ImagesiftBot","magpie-crawler","Meta-ExternalAgent","meta-externalagent","NewsNow","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Quora-Bot","Scrapy","TurnitinBot"]}
{"domain":"crossthegame.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cloudflareaccess.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sudoku.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AmazonAdBot","*"]}
{"domain":"f2pool.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dwin1.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"milfporn.tv","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"scryfall.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zillowstatic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vedantu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"serebii.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nuance.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"jmty.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","bingbot","Baiduspider","trovitBot","grapeshot"]}
{"domain":"uni-stuttgart.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","Xenu Link Sleuth","Yandex","ICCrawler - iCjobs","Web Downloader/3.0","MJ12bot","AhrefsBot"]}
{"domain":"fuckmoral.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","bingbot","YandexBot","SemrushBot","dotbot","ia_archiver","MJ12bot"]}
{"domain":"yesporn.vip","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ghostery.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","attempts":["ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"hentaifox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"gunbroker.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"dnsservice.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_PACKET_LENGTH_TOO_LONG","attempts":["ERR_SSL_PACKET_LENGTH_TOO_LONG","UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"ecobee.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lifewire.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"tube8.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":23,"stanza_tokens":["*","Amazonbot","Anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","Cohere-ai","DataForSeoBot","FacebookBot","GPTBot","ImagesiftBot","Magpie-crawler","Omgili","Omgilibot","Peer39_crawler","Peer39_crawler/1.0","PerplexityBot","YouBot"]}
{"domain":"zalando.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Pinterestbot","*"]}
{"domain":"sex-studentki.sale","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gwars.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"censor.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mbl.is","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"15min.lt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":46,"stanza_tokens":["*","facebookexternalhit","AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","Applebot-Extended","Brightbot 1.0","Bytespider","CCBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Diffbot","DuckAssistBot","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","imgproxy","ISSCyberRiskCrawler","Kangaroo Bot","NovaAct","omgili","omgilibot","Operator","PanguBot","Perplexity-User","PerplexityBot","PetalBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","TikTokSpider","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot","BLEXBot","Meltwater","grapeshot"]}
{"domain":"nfpa.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Storebot-Google","Googlebot","Googlebot-image"]}
{"domain":"ilmattino.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":28,"stanza_tokens":["*","uMBot","uMBot-LN","uMBot-IC","Arianna","Arianna News","Arianna Web","Arianna Robot","GPTBot","Google-Extended","anthropic-ai","CCBot","Omgilibot","cohere-ai","ChatGPT-User","OAI-SearchBot","PerplexityBot","Amazonbot","YouBot","Claude-Web","Claudebot","Claude-User","Claude-SearchBot","Diffbot","Bytespider","ImagesiftBot","Meltwater","Seekr"]}
{"domain":"e-planning.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"paheal.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rackspacecloud.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"metacritic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":15,"stanza_tokens":["AhrefsBot","AwarioRssBot","AwarioSmartBot","barkrowler","BLEXBot","CCBot","DataForSeoBot","dotbot","GPTBot","ImagesiftBot","MJ12bot","SemrushBot","TurnitinBot","*","AmazonAdBot"]}
{"domain":"kiabi.com","fetch_ok":false,"usable":false,"reason":"other","status":406,"error":null,"attempts":[406,406]}
{"domain":"eventim.de","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nishinippon.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":45,"stanza_tokens":["AmazonAdBot","GPTBot","CCBot","ChatGPT-User","Google-Extended","anthropic-ai","cohere-ai","omgili","omgilibot","ICC-Crawler","Applebot-Extended","ClaudeBot","Claude-Web","PerplexityBot","Perplexity-ai","Bytespider","Diffbot","FacebookBot","OAI-SearchBot","MJ12bot","PiplBot","Meta-ExternalAgent","Timpibot","Webzio","Webzio-extended","Perplexity-User","Claude-SearchBot","Claude-User","AmazonBot","cohere-training-data-crawler","magpie-crawler","meta-externalfetcher","Scrapy","DuckAssistBot","Google-CloudVertexBot","AI2Bot","AI2Bot-Dolma","img2dataset","YouBot","SeekrBot","OpenindexSpider","Gensparkbot","SBIntuitionsBot","SBIntuitions-SearchBot","*"]}
{"domain":"nordkurier.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":15,"stanza_tokens":["*","Google-Extended","GPTBot","ChatGPT-User","CCBot","ClaudeBot","Bytespider","Amazonbot","Applebot-Extended","ia_archiver","Arquivo-web-crawler","anthropic-ai","OAI-SearchBot","PerplexityBot","Claude-Web"]}
{"domain":"in.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ontario.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"but.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","TurnitinBot","ConveraCrawler","QuepasaCreep","Jetbot","TwengaBot"]}
{"domain":"shop-apotheke.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","AdIdxBot","*"]}
{"domain":"awsdns-01.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"lifo.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"instapaper.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"network-lc.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ECONNRESET"]}
{"domain":"newlook.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","AdsBot-Google","meta-externalagent","facebookexternalhit"]}
{"domain":"mudah.my","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["ia_archiver","Speobot","*"]}
{"domain":"york.ac.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["linecker/*","*","SiteimproveBot-Crawler"]}
{"domain":"smotrim.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"r7.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["proximic","SemrushBot","Twitterbot","SemrushBot-SA","Starkbot","Scrapy","VoluumDSP-content-bot","Seekport Crawler","GPTBot","Google-Extended","*"]}
{"domain":"copart.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.com.pr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"bhaskar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":28,"stanza_tokens":["*","CCBot","GPTBot","Google-Extended","anthropic-ai","Omgilibot","Omgili","FacebookBot","Diffbot","Bytespider","ImagesiftBot","cohere-ai","PerplexityBot","ClaudeBot","Claude-Web","Meta-ExternalAgent","Meta-ExternalFetcher","Applebot","Applebot-Extended","Amazonbot","ChatGPT-User","OAI-SearchBot","Gemini-Deep-Research","GoogleOther","Google-CloudVertexBot","Claude-User","Claude-SearchBot","Perplexity-User"]}
{"domain":"oem.com.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","grapeshot","AmazonAdBot"]}
{"domain":"about.google","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"albumoftheyear.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"emptyhammock.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wow.xxx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rmf24.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Mediapartners-Google"]}
{"domain":"rp-online.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":44,"stanza_tokens":["Amazonbot","anthropic-ai","Applebot","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","CloudVertexBot","cohere-ai","DataForSeoBot","DeepSeek","DeepSeekBot","Diffbot","FacebookBot","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","GPTBot","ImagesiftBot","magpie-crawler","Meta-ExternalAgent","meta-externalagent","NewsNow","news-please","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Perplexity-User","Quora-Bot","Scrapy","TurnitinBot","YandexAdditional","YandexAdditionalBot","AmazonAdBot","*"]}
{"domain":"mattel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kff.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"meitudata.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"podscribe.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":30,"stanza_tokens":["AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","DuckAssistBot","FacebookBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","img2dataset","Meta-ExternalAgent","MyCentralAIScraperBot","omgili","omgilibot","Quora-Bot","TikTokSpider","YouBot","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","*"]}
{"domain":"ndm9.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"wordplays.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":31,"stanza_tokens":["*","Yandex","proximic","ia_archiver","linkdexbot","grapeshot","AhrefsBot","Baiduspider","Clickagy","Clickagy Intelligence Bot v2","Genieo","MJ12bot","ADmantX","crystalsemantics","getintent","Amazon.CloudFront","yoozBot","baidu","WebIndex","Bombora","Sogou","Experibot","HybridBot","Slurp","expo9","CCBot","RealPlayer","weborama","masscan","Cliqzbot","Leikibot"]}
{"domain":"geocities.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"ole.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"privacy.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Google-Extended","ChatGPT-User","Anthropic-AI","Manus"]}
{"domain":"ugr.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"51cg1.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"appboycdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wtr-lab.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"valve.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"espn.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":11,"stanza_tokens":["claritybot","GPTBot","Google-Extended","CCBot","ChatGPT-User","anthropic-ai","Omgilibot","Omgili","FacebookBot","Bytespider","*"]}
{"domain":"filmstarts.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","seekbot","Exabot","Slurp","Cliqzbot","Arquivo-web-crawler","AI2Bot","ChatGLM","ChatGLM-spider","cohere-training-data-crawler","Diffbot","OmigiliBot","PetalBot","YouBot"]}
{"domain":"fortnitetracker.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":12,"stanza_tokens":["*","Yandex","Bingbot","peer39_crawler","peer39_crawler/1.0","ChatGPT-User","GPTBot","Applebot-Extended","ClaudeBot","DataForSeoBot","meta-externalagent","AmazonAdBot"]}
{"domain":"businesstech.co.za","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"boots.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AdsBot-Google"]}
{"domain":"nzz.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":48,"stanza_tokens":["*","Googlebot-News","MegaIndex.ru/2.0","BLP_bbot/0.1","Slurp","GPTBot","ChatGPT-User","Google-Extended","CCBot","anthropic-ai","Bytespider","ClaudeBot","Claude-Web","cohere-ai","DataForSeoBot","Diffbot","FacebookBot","magpie-crawler","NewsNow","news-please","omgilibot","PerplexityBot","Amazonbot","Applebot-Extended","AwarioRssBot","AwarioSmartBot","FriendlyCrawler","ImagesiftBot","Meta-ExternalAgent","meta-externalagent","OAI-SearchBot","peer39_crawler","peer39_crawler/1.0","Quora-Bot","Scrapy","TurnitinBot","ChatGPT","ICC-Crawler","omgili","Operator","Perplexity-User","PetalBot","TikTokSpider","Timpibot","Webzio-Extended","MistralAI-User","Anchor Browser","Meta-ExternalFetcher"]}
{"domain":"unian.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","googlebot","yandex","googlebot-image","googlebot-video","yandexvideo"]}
{"domain":"t24.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","facebookexternalhit","Facebot"]}
{"domain":"realizationnewestfangs.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"moneta.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":2,"stanza_tokens":["*","GPTBot"]}
{"domain":"vecernji.hr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"telegram.im","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uni-tuebingen.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","PetalBot"]}
{"domain":"n1info.rs","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"meduza.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"googlecode.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"1cbit.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"letras.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"ky.gov","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_UNSUPPORTED_PROTOCOL","attempts":["ERR_SSL_UNSUPPORTED_PROTOCOL","ERR_SSL_UNSUPPORTED_PROTOCOL"]}
{"domain":"s-onetag.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","EAI_AGAIN"]}
{"domain":"cloud.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","Googlebot","*"]}
{"domain":"xxxshame.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","bingbot","YandexBot","SemrushBot","dotbot","ia_archiver","MJ12bot"]}
{"domain":"icbc.com.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ERR_SSL_UNSAFE_LEGACY_RENEGOTIATION_DISABLED",404]}
{"domain":"moneyforward.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-17.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"securebrowser.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nflximg.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dnsitalia.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"abola.pt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"delish.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AmazonAdBot","Google-InspectionTool","ChatGPT-User","GPTBot","OAI-SearchBot","*"]}
{"domain":"rrpicturearchives.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["ClaudeBot","anthropic-ai","GPTBot","Bingbot","SemrushBot","SiteAuditBot","Amazonbot","amazonbot","Meta-ExternalAgent","meta-externalagent","meta-webindexer","*"]}
{"domain":"vidoomy.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"iqzonertb.live","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"smile.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"47news.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["bingbot","msnbot","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Diffbot","FacebookBot","Google-Extended","GPTBot","ICC-Crawler","magpie-crawler","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","omgili","omgilibot","Perplexity","Perplexity-ai","Perplexity-User","PerplexityBot","Scrapy","Timpibot","Webzio-Extended","Gensparkbot","*"]}
{"domain":"logi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"farming-simulator.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"linkssp.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"userreport.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pravda.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","*"]}
{"domain":"vodafone.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"businessinsider.com.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"boticario.com.br","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"awsdns-61.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"corsair.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","CazoodleBot","MJ12bot","dotbot/1.0","Gigabot","Semrushbot-SA","Semrushbot-SI","Caliperbot/1.0","Googlebot","Googlebot-image","OAI-SearchBot"]}
{"domain":"lance.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot-News"]}
{"domain":"gamifyspace.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"citrixworkspacesapi.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-55.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"videojs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"go2cloud.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",404]}
{"domain":"nouvelobs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":["OAI-SearchBot"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":314,"stanza_tokens":["Adsbot-Google","AdsBot-Google-Mobile","APIs-Google","Applebot","Bingbot","exabot","Googlebot","Googlebot-Image","Googlebot-Mobile","Googlebot-News","Googlebot-Video","ia_archiver","Mediapartners-Google","Proximic","Slurp","Twitterbot","sistrix","Flipboard","FlipboardProxy","008","Alexibot","AlvinetSpider","Antenne Hatena","anthropic-ai","ApocalXExplorerBot","asterias","BackDoorBot/1.0","BizInformation","Black Hole","BlowFish/1.0","BotALot","BuiltBotTough","Bullseye/1.0","BunnySlippers","CCBot","Cegbfeieh","ChatGPT-User","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","Claude-Web","ClaudeBot","CopyRightCheck","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","DISCo Pump 3.1","DittoSpyder","dotbot","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","ExtractorPro","FacebookBot","Flamingo_SearchEngine","Foobot","Google-Extended","GPTBot","g2reader-bot/1.0","Harvest/1.5","hloader","httplib","HTTrack","HTTrack 3.0","humanlinks","Igentia","InfoNaviRobot","JennyBot","JikeSpider","Kenjin Spider","LexiBot","libWeb/clsHTTP","LinkextractorPro","LinkScan/8.1a Unix","LinkWalker","lwp-trivial","lwp-trivial/1.34","Mata Hari","MediafedMetrics","Microsoft URL Control - 5.01.4511","Microsoft URL Control - 6.00.8169","MIIxpc","MIIxpc/4.2","Mister PiX","MLBot","moget","moget/2.1","MS Search 4.0 Robot","MS Search 5.0 Robot","Naverbot","NetAnts","NetAttache","NetMechanic","NICErsPRO","Nutch","Offline Explorer","omgili","omgilibot"]}
{"domain":"timesonline.co.uk","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"zerogpt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wixmp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AdsBot-Google-Mobile","AdsBot-Google","PetalBot","dotbot","AhrefsBot"]}
{"domain":"uwo.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["UWO-InktomiSearch","*"]}
{"domain":"trusteer.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"cargocollective.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":["ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":19,"stanza_tokens":["OAI-SearchBot","GPTBot","ChatGPT-User","Googlebot","Google","Mediapartners-Google","bingbot","bing","msnbot","msn","Slurp","Yandex","Facebot","Twitterbot","ia_archiver","naver","NaverBot","Yeti","*"]}
{"domain":"eldebate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"splus.ir","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"mq.edu.au","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"awsdns-53.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"higgsfield.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-Web","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"halodoc.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mcl.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"doodle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AmazonAdBot","*"]}
{"domain":"uptimerobot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rpi.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mn.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Ultraseek","VSE/1.0"]}
{"domain":"libertatea.ro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","googlebot-news","ClaudeBot"]}
{"domain":"bet9ja.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"e.leclerc","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"impress.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"amung.us","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"refinery29.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":134,"stanza_tokens":["GPTBot","ChatGPT-User","OAI-SearchBot","ChatGPT Agent","Operator","OpenAI","ClaudeBot","anthropic-ai","Claude-SearchBot","Claude-User","Claude-Web","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","Google-CloudVertexBot","CloudVertexBot","Google-NotebookLM","NotebookLM","GoogleAgent-Mariner","Gemini-Deep-Research","Google-Firebase","Applebot-Extended","Meta-ExternalAgent","meta-externalagent","Meta-ExternalFetcher","meta-externalfetcher","meta-webindexer","FacebookBot","facebookexternalhit","Bytespider","TikTokSpider","Amazonbot","Amzn-SearchBot","Amzn-User","AmazonBuyForMe","amazon-kendra","PerplexityBot","Perplexity-User","DeepSeekBot","MistralAI-User","MistralAI-User/1.0","CCBot","cohere-ai","cohere-training-data-crawler","AzureAI-SearchBot","bedrockbot","Bravebot","DuckAssistBot","PhindBot","YouBot","PetalBot","Diffbot","iAskBot","iaskspider","iaskspider/2.0","kagi-fetcher","LinerBot","LinkupBot","IbouBot","PanguBot","ChatGLM-Spider","Manus-User","NovaAct","Devin","TwinAgent","Scrapy","Crawl4AI","Crawlspace","FirecrawlAgent","img2dataset","LAIONDownloader","laion-huggingface-processor","Cloudflare-AutoRAG","TavilyBot","omgili","omgilibot","AddSearchBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","Andibot","Anomura","atlassian-bot","Awario","bigsur.ai","Brightbot 1.0","BuddyBot","Channel3Bot","Cotoyogi","Datenbank Crawler","Echobot Bot","EchoboxBot","Factset_spyderbot","FriendlyCrawler","ICC-Crawler","ImagesiftBot","imageSpider","ISSCyberRiskCrawler"]}
{"domain":"tycsports.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["facebookexternalhit","facebookcatalog","Twitterbot","LinkedInBot","WhatsApp","Pinterest","TelegramBot","redditbot","*"]}
{"domain":"caesars.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"asics.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"garanteprivacy.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Googlebot","Bingbot","Baiduspider","Twitterbot"]}
{"domain":"unstoppabledomains.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"majordomo.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","GoogleBot","Yandex","GPTBot","ClaudeBot","PerplexityBot","OAI-SearchBot","Googlebot-Extended"]}
{"domain":"amspbs.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"searchspring.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"zdbb.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"klix.ba","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","Googlebot","Googlebot-Image","facebookexternalhit","facebookcatalog","meta-externalagent","meta-externalfetcher","Facebot","SemrushBot","AhrefsBot","googlebot-ads","mediapartners-google"]}
{"domain":"batch.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Baiduspider","*"]}
{"domain":"ilrestodelcarlino.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":9,"stanza_tokens":["*","Applebot-Extended","Bytespider","ClaudeBot","Claude-Web","anthropic-ai","archive.org_bot","CCBot","Arquivo-web-crawler"]}
{"domain":"freeones.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":28,"stanza_tokens":["Linguee","Slurp","ia_archiver","GPTBot","ChatGPT-User","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Claude-Web","cohere-ai","Diffbot","FacebookBot","FriendlyCrawler","ImagesiftBot","img2dataset","Meta-ExternalAgent","OAI-SearchBot","omgili","omgilibot","PerplexityBot","Scrapy","Timpibot","VelenPublicWebCrawler","YouBot","*"]}
{"domain":"notepad-plus-plus.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"abema.tv","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"awsdns-36.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"angi.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"finalfantasyxiv.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["DEPTH_ZERO_SELF_SIGNED_CERT",404]}
{"domain":"nic.kw","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"centrum24.pl","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"stardewvalleywiki.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":11,"stanza_tokens":["MJ12bot","SemrushBot","Amazonbot","GoogleOther","OAI-SearchBot","ClaudeBot","GPTBot","ImagesiftBot","Meta-ExternalAgent","meta-webindexer/1.1","*"]}
{"domain":"ngenix-dns.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"smartcasttv.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"ge.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ekitan.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","CriteoBot/0.1","AmazonAdBot","Googlebot","ClaudeBot"]}
{"domain":"wal-mart.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Adsbot-Google","Mediapartners-Google","Slurp"]}
{"domain":"sejda.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"partslink24.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Bytespider","AhrefsBot","Linguee"]}
{"domain":"ukrainealarm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AdsBot-Google-Mobile","AdsBot-Google","PetalBot","dotbot","AhrefsBot"]}
{"domain":"novotelecom.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ECONNRESET"]}
{"domain":"netcoresmartech.com","fetch_ok":false,"usable":false,"reason":"other","status":500,"error":null,"attempts":[500,500,500]}
{"domain":"galaxydata.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","GoogleBot","Yandex"]}
{"domain":"reson8.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"ucm.es","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"tudocelular.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.lk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"netdna-ssl.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dunelm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot-Image","AdsBot-Google","AdsBot-Google-Mobile"]}
{"domain":"papy.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sporza.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":21,"stanza_tokens":["*","AI2Bot","Amazonbot","Applebot-Extended","anthropic-ai","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","Diffbot","DuckAssistBot","FacebookBot","Google-Extended","GPTBot","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","PerplexityBot","PerplexityUser"]}
{"domain":"muzofond.fm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","*","ia_archiver"]}
{"domain":"cloudbackup.management","fetch_ok":false,"usable":false,"reason":"other","status":401,"error":null,"attempts":[401,530]}
{"domain":"netcraze.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_GET_ISSUER_CERT_LOCALLY","attempts":["UNABLE_TO_GET_ISSUER_CERT_LOCALLY","UNABLE_TO_GET_ISSUER_CERT_LOCALLY"]}
{"domain":"ets.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-27.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mackolik.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":6,"stanza_tokens":["*","GPTBot","Amazonbot","Google-Extended","CCBot","anthropic-ai"]}
{"domain":"hpcl.co.in","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"esic.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"mpsuadv.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"adsmeasurement.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"geodns.me","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"spacebattles.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"pokemondb.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","Yandex","SindiceBot","CCBot","wget","WebReaper","AhrefsBot"]}
{"domain":"wb.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bpmlt.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"livescore.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"ui-dns.biz","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"archiveofourown.gay","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"jwplatform.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"dkitrxmdwoqruvsi.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"palmettostatearmory.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"jet2holidays.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AhrefsBot"]}
{"domain":"fakt.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nuget.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.com.cy","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"parliament.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"sanspo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":50,"stanza_tokens":["*","AI2Bot","anthropic-ai","Applebot-Extended","Arquivo-web-crawler","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","coccocbot-web","cohere-ai","cohere-training-data-crawler","Crawlspace","Diffbot","DuckAssistBot","FriendlyCrawler","Gensparkbot","Google-CloudVertexBot","Google-Extended","GPTBot","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","magpie-crawler","MoodleBot","Netvibes","NewsNow","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","Scrapy","Seekr","SemrushBot","Timpibot","Turnitin","TurnitinBot","VelenPublicWebCrawler","Webzio-Extended","yacy","yacybot","YouBot","Baiduspider"]}
{"domain":"forbes.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"hsbc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yuga.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","YandexImages"]}
{"domain":"sport.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":30,"stanza_tokens":["AiHitBot","BLEXBot","CareerBot","CCBot","dotbot","Google-Extended","GPTBot","grapeshot","iCjobs","linkdexbot","magpie-crawler","MegaIndex","MJ12bot","proximic","QuerySeekerSpider","scrapy","scrapybot","SemrushBot","sentibot","SEOkicks-Robot","TkBot","trendkite-akashic-crawler","Vagabondo","WBSearchBot","AhrefsBot","Baiduspider","YandexBot","Yahoo! Slurp","*","facebookexternalhit"]}
{"domain":"ksyun.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"znews.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["facebookexternalhit","Facebot","*"]}
{"domain":"hypernode.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ENOTFOUND"]}
{"domain":"sport.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"impactcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"xbox-dns.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"alodokter.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","Googlebot","Mediapartners-Google","Adsbot-Google","Googlebot-Image","Googlebot-Mobile","Domain Re-Animator Bot","ClaudeBot","Claude-User","Claude-SearchBot","GPTBot","OAI-SearchBot","ChatGPT-User","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"bling.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cust-service.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"tryst.link","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["Googlebot","Googlebot-Image","YandexBox","DuckDuckBot","Slurp","MSNBot","MSNBot-Media","Bingbot","SemrushBot","AhrefsBot","Twitterbot","*"]}
{"domain":"capital.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":6,"stanza_tokens":["*","BLEXBot","Mediapartners-Google","Googlebot-Mobile","GPTBot","CCBot"]}
{"domain":"net.hr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Screaming Frog SEO Spider","facebookexternalhit","*"]}
{"domain":"kabutan.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"expresso.pt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":80,"stanza_tokens":["AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","Andibot","anthropic-ai","Applebot","Applebot-Extended","Awario","bedrockbot","Brightbot 1.0","Bytespider","CCBot","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Datenbank Crawler","Devin","Diffbot","DuckAssistBot","FacebookBot","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NovaAct","OAI-SearchBot","omgili","omgilibot","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot","Scrapy","Sidetrade indexer bot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","wpbot","YandexAdditional","YandexAdditionalBot","YouBot","*"]}
{"domain":"pxltag.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"digitalaudience.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ddddns.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"eurosport.fr","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"telcel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","GPTBot","anthropic-ai","ClaudeBot","PerplexityBot","Google-Extended","BingBot"]}
{"domain":"discover.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"word.tips","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["Googlebot","BLEXBot","dotbot","semrushbot","ahrefsbot","mj12bot","barkrowler","siteliner","linkbot","cliqzbot"]}
{"domain":"manga18.club","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":19,"stanza_tokens":["*","Teleport","TeleportPro","EmailCollector","EmailSiphon","WebBandit","WebZIP","WebReaper","WebStripper","Web Downloader","WebCopier","Offline Explorer Pro","Offline Explorer","HTTrack Website Copier","Offline Commander","Leech","WebSnake","BlackWidow","HTTP Weazel"]}
{"domain":"kaptcha.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cloudscdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dwcdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ldplayer.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Bingbot","msnbot"]}
{"domain":"askubuntu.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":418,"error":null,"attempts":[418,418]}
{"domain":"hosted-by-discourse.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"salesforce-sites.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["googlebot","*"]}
{"domain":"marketing-hub.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ECONNRESET"]}
{"domain":"awsdns-15.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ks-live.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"transcend.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gdt.gov.vn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"indiapost.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"hingeprod.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pa-cd.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-09.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"w390.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNREFUSED"]}
{"domain":"hanime.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"qoo10.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","Googlebot","Bingbot","Slurp","DuckDuckBot","Meta-ExternalAds","OAI-SearchBot","GPTBot","ChatGPT-User","ClaudeBot","Claude-User","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider"]}
{"domain":"gitea.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"hostpapa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["^Java","^Jakarta","libwww","Snoopy","YandexSomething/1.0","Mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)","Mediapartners-Google","Googlebot-Image","Adsbot-Google","Googlebot-Mobile","SemrushBot","SemrushBot-SA","*"]}
{"domain":"nrc.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":84,"stanza_tokens":["*","magpie-crawler","AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","Andibot","anthropic-ai","Applebot-Extended","Awario","bedrockbot","Brightbot 1.0","Bytespider","CCBot","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Datenbank Crawler","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","meta-externalagent","Meta-ExternalAgent","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NovaAct","OAI-SearchBot","omgili","omgilibot","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","SummalyBot","Thinkbot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","wpbot","YandexAdditional","YandexAdditionalBot","YouBot"]}
{"domain":"google.co.zm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"myherbalife.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"envoyer.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"utwente.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"president.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cmjornal.pt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":8,"stanza_tokens":["OpenAI-Crawler","Googlebot-Bard","GPTBot","ChatGPT-User","Google-Extended","CCBot","facebookexternalhit","*"]}
{"domain":"brother.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"kufar.by","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","schibstedsokbot","facebookexternalhit","Facebot"]}
{"domain":"clck.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wknd.ai","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"epson.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["*","CazoodleBot","MJ12bot","dotbot/1.0","Gigabot","SEMrushBot","Yandex","Baiduspider","DotBot","MegaIndex.ru","BLEXBot","SEOkicks-Robot","Exabot","AlphaBot","Sogou web spider","Ezooms","LinkpadBot","WebMeUp"]}
{"domain":"arvixe.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thredup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","bingbot","GoogleOther","MJ12bot","Pinterestbot","YandexBot"]}
{"domain":"linodeusercontent.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"creativefabrica.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.com.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"baidu-itm.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"icrc.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sojern.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ohio-state.edu","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"betterstackdata.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"anl.gov","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"pm-serv.co","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","EAI_AGAIN"]}
{"domain":"awsdns-14.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"picrew.me","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"spacedaily.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"asp.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"bayer.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"stroyserviceru.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"auchan.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","Mediapartners-Google","Googlebot-Image","Storebot-Google","Sentibot","*"]}
{"domain":"bbva.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"catracalivre.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot-News"]}
{"domain":"instiz.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"google.is","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"boulanger.com","fetch_ok":false,"usable":false,"reason":"other","status":400,"error":null,"attempts":[400,400]}
{"domain":"edmentum.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"ecoticias.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","grapeshot","AhrefsBot","Rogerbot","Dotbot","MJ12bot","SemrushBot","SemrushBot-SA","PetalBot","Bytespider"]}
{"domain":"optum.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"forbesjapan.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nettavisen.no","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":85,"stanza_tokens":["*","CCBot","GPTBot","anthropic-ai","Google-Extended","Amazonbot","Claude-SearchBot","Meta-ExternalAgent","Bytespider","ClaudeBot","Timpibot","OAI-SearchBot","Claude-User","Applebot-Extended","AI2Bot","Ai2Bot-Dolma","Anomura","bedrockbot","Channel3Bot","ChatGLM-Spider","cohere-ai","cohere-training-data-crawler","Crawl4AI","Crawlspace","DeepSeekBot","Diffbot","FirecrawlAgent","FriendlyCrawler","GoogleOther","GoogleOther-Image","GoogleOther-Video","GrokBot","ICC-Crawler","img2dataset","ImagesiftBot","Kangaroo Bot","KunatoCrawler","laion-huggingface-processor","LAIONDownloader","meta-webindexer","MyCentralAIScraperBot","omgili","omgilibot","PanguBot","SBIntuitionsBot","Scrapy","Spider","TerraCotta","TikTokSpider","VelenPublicWebCrawler","Webzio-Extended","xAI-Grok","YandexAdditional","YandexAdditionalBot","Google-CloudVertexBot","PerplexityBot","Amzn-SearchBot","AzureAI-SearchBot","Bravebot","CloudVertexBot","Grok-DeepSearch","iAskBot","iaskspider","iaskspider/2.0","LinerBot","LinkupBot","PhindBot","TavilyBot","WRTNBot","YouBot","ChatGPT-User","Perplexity-User","MistralAI-User","Meta-ExternalFetcher","AmazonBuyForMe","Amzn-User","ChatGPT Agent","Devin","Gemini-Deep-Research","Google-NotebookLM","GoogleAgent-Mariner","Manus-User","NovaAct","Operator","Claude-Web"]}
{"domain":"i-ready.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nic.ir","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"24ur.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ecer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"biccamera.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"arubainstanton.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["CERT_HAS_EXPIRED",403]}
{"domain":"allfreenovel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"leadsquared.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gnezdo.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"edge2befaster.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"komiku.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"marmiton.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Mediapartners-Google*","grapeshot","YahooSeeker*","MSNBOT_Mobile","AmazonAdBot","*"]}
{"domain":"lindependant.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":240,"stanza_tokens":["*","5emeRue","5erue","adequat","adequat-systems","Alexibot","AlvinetSpider","Amazonbot","AmiSoftware","Antenne Hatena","anthropic-ai","ApocalXExplorerBot","Argus","Ask n read","asknread.com","asterias","Augure","auramundi","BackDoorBot/1.0","BizInformation","Black Hole","Bloodhound","Bloomberg","BlowFish/1.0","BotALot","BraveBot","BuiltBotTough","Bullseye/1.0","BunnySlippers","Bytespider","CCBot","Cegbfeieh","ChatGPT-User","OAI-SearchBot","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","Cision","Claude-Web","ClaudeBot","coexel","cohere-ai","ConveraCrawler","CopyRightCheck","Corporama","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","cydralspider","Diffbot","Digimind","DISCo Pump 3.1","DittoSpyder","dotbot","Download Ninja","downloadexpress","EDD","ellisphere","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","eureka","Europresse","Explore","ExtractorPro","FacebookBot","Factiva","Fasterfox","Fetch","Flamingo_SearchEngine","flipboard","Foobot","gammaSpider","Google-Extended","GPTBot","grub-client","Harvest/1.5","hloader","httplib","HTTrack","HTTrack 3.0","humanlinks","ia_archiver","ia_archiver-web.archive.org","Igentia","indexer","InfoNaviRobot","infoseek","JennyBot","Jetbot","JikeSpider","k2spider","Kantar","kbcrawl","Kenjin Spider","Knowings","larbin","leadbox"]}
{"domain":"peakgames.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":5,"stanza_tokens":["*","ImagesiftBot","ClaudeBot","Claude-User","Claude-SearchBot"]}
{"domain":"flightrising.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":9,"stanza_tokens":["yahoo-mmcrawler","psbot","asterias","yahoo-blogs/v3.9","*","GPTBot","ChatGPT-User","ClaudeBot","Bytespider"]}
{"domain":"suning.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ERR_SSL_UNSAFE_LEGACY_RENEGOTIATION_DISABLED",404]}
{"domain":"mesh.ad.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sfndns.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"esper.cloud","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ENOTFOUND"]}
{"domain":"destentor.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":30,"stanza_tokens":["*","Twitterbot","GPTBot","ChatGPT-User","CCBot","anthropic-ai","Amazonbot","Applebot-Extended","Bytespider","ClaudeBot","Claude-Web","cohere-ai","Diffbot","Google-Extended","Meta-ExternalFetcher","PerplexityBot","YouBot","DeepSeekBot","MistralAI-User","MistralAI-User/1.0","PetalBot","Timpibot","YandexAdditional","YandexAdditionalBot","YandexBot","YandexRenderBot","Perplexity-User","coccocbot-web","OAI-SearchBot","Claude"]}
{"domain":"vewd.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"messefrankfurt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bluewin.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":21,"stanza_tokens":["*","facebookexternalhit","FacebookBot","GDdash SEO Spider/2.0","Flamingo_SearchEngine","ExaBot","dotbot","genieo","crystalsemantics","grapeshot","SEMrushBot","Scrapy","AhrefsBot","DataForSeoBot","ClaudeBot","Claude-Web","cohere-ai","Diffbot","magpie-crawler","NewsNow","news-please"]}
{"domain":"vhostgo.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"awsdns-44.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"ext-twitch.tv","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"vu.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-60.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"ors.at","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"sinajs.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"ddnsfree.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"tkmaxx.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"google.ee","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"gotoresolve.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cgg.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ozon.by","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"soccer365.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"motorola.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"sandai.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"danas.rs","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"google.fm","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"monotaro.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","UND_ERR_CONNECT_TIMEOUT","TimeoutError"]}
{"domain":"examiner.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"gcore.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"mgaru.dev","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jfrog.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"secomtrust.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"slobodnadalmacija.hr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"apps.chrome","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bellsouth.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"wi.gov","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"rb.gy","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"ndmais.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","PetalBot","CCBot","dotbot"]}
{"domain":"orangedata.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thepaper.cn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"udmserve.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"picks.my","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["Googlebot","Yeti","Daumoa","ChatGPT-User","GPTBot","grapeshot","*"]}
{"domain":"deped.gov.ph","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"heroeswm.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"evite.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"zalando.it","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"financialexpress.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"olxcdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ballotpedia.org","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"runwayml.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ewg.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"opt360.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bergfex.at","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",429]}
{"domain":"licindia.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gyazo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nationalreview.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"pornzog.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"steelseries.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"flowgpt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ridibooks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Bytespider","meta-externalagent"]}
{"domain":"google.mv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"googlepages.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ECONNRESET"]}
{"domain":"psychcentral.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":28,"stanza_tokens":["GPTBot","Applebot-Extended","anthropic-ai","Bytespider","CCBot","ClaudeBot","Diffbot","FacebookBot","Meta-ExternalAgent","omgili","Timpibot","Webzio-Extended","Mediapartners-Google","Adsbot-Google","AmazonAdbot","Amazonbot","Amzn-SearchBot","Amzn-User","amazon-kendra","amazon-QBusiness","bedrockbot","aws-quick-on-behalf-of-","AmazonProductDiscoverybot","AmazonBuyForMe","AmazonSellerInitiatedListing","NovaAct","DDM-DCipher/1.0.7","*"]}
{"domain":"apollo.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":29,"stanza_tokens":["*","anthropic-ai","ChatGPT-User","ClaudeBot","cohere-ai","GPTBot","OAI-SearchBot","PerplexityBot","CCbot","Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News","AdsBot-Google","AdsBot-Google-Mobile","Google-Extended","Bingbot","DuckDuckGo","Applebot","facebot","Yandex","Baiduspider","Sogou","Naverbot","MojeekBot","PetalBot","AhrefsBot","SemrushBot","MJ12bot"]}
{"domain":"bolt.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","OAI-SearchBot"]}
{"domain":"krakow.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["AhrefsBot"]}
{"domain":"netmarble.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ticketmaster.com.mx","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"lesnumeriques.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"btttag.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"freshservice.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"speakerdeck.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"complex.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":31,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","Omgilibot","Omgili","DeepSeekBot","DeepSeek","DuckAssistBot","OAI-Searchbot","ChatGPT-User","Perplexity","discobot","dotbot","TurnitinBot","magpie-crawler","yacybot","Timpibot","Nutch","grapeshot","adsbot-google","AhrefsBot","AhrefsSiteAudit","MJ12bot","Pinterest"]}
{"domain":"getresponse.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hostmonster.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"heapanalytics.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"firebasedatabase.app","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"wall-box.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"intercars.eu","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"awsdns-55.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"rtb.mx","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"extcam.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"lenta.tech","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"kas-labs.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"moe.gov.sa","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"darty.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"google.bs","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"creditmutuel.fr","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ernet.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"dotloop.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"mcdonalds.co.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"gsp.ro","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"borsaitaliana.it","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"telegraf.rs","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"vhx.tv","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"pitchfork.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"magicbricks.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nrdc.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"turbosquid.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ad-stir.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"awsdns-21.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"awsdns-04.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"pingdom.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"domain.com.au","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"poynter.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"calm.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"baitoru.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"jpmorganchase.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"shein.com.mx","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"mcas.ms","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"zani64.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"html-load.cc","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wikiwiki.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"appleinsider.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unige.ch","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"climatempo.com.br","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"jr-odekake.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"salthasher.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"usaid.gov","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"sisal.it","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"confluent.cloud","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"fca.org.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.ba","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"huffingtonpost.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":21,"stanza_tokens":["*","meta-externalagent","Meta-ExternalFetcher","ChatGPT-User","OAI-SearchBot","Bytespider","DuckAssistBot","Perplexity-User","Timpibot","ClaudeBot","Amazonbot","GPTBot","PerplexityBot","CCBot","FacebookBot","Ai2Bot-Dolma","Claude-SearchBot","Claude-User","omgilibot","TtoYouBot","WebGPTBot"]}
{"domain":"attentivemobile.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"articulate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jcb.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"indiewire.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":25,"stanza_tokens":["*","anthropic-ai","cohere-ai","omgili","omgilibot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ChatGPT","DeepSeek","DeepSeekBot","GPTBot","OAI-SearchBot","ClaudeBot","Claude-Web","Diffbot","FacebookBot","Google-Extended","Meta-ExternalAgent","Meta-ExternalFetcher","PerplexityBot","Timpibot","Webzio-Extended","YouBot"]}
{"domain":"usbrowserspeed.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tsa.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","PetalBot"]}
{"domain":"monopolygo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"masrawy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"donedeal.ie","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"popsugar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":["GPTBot"],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":105,"stanza_tokens":["Amazonbot","Applebot","GPTBot","A6-Indexer","AhrefsBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot-Extended","Baiduspider","betaBot","BLEXBot","botify","brands-bot-logo","Brightbot","Bytespider","calculon","CCBot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","collection@infegy.com","Cotoyogi","Crawl4AI","CrunchBot","deepcrawl","DeepSeekBot","Diffbot","DirBuster","DTS","DuckAssistBot","EasouSpider","FacebookBot","FirecrawlAgent","FriendlyCrawler","GlutenCrawler","Google-CloudVertexBot","Google-Extended","GrapeshotCrawler","iaskspider","ICC-Crawler","ImagesiftBot","img2dataset","JackCrawl","Jetslide","linkapedia_bot","Mediatoolkitbot","MegaIndex","meta-externalagent","meta-externalfetcher","Microsoft BITS","Microsoft URL Control","Microsoft-WebDAV-MiniRedir","MistralAI-User","news-please","Novellum","NTENTbot","Nutch","omgili","Omgilibot","OpenLinkProfiler","Perplexity-User","PerplexityBot","PetalBot","Primalbot","python-requests","Re-Animator","Riddler","SafeDNSBot","SBIntuitionsBot","Scrapy","SearchAtlas","SemanticScholarBot","SemrushBot","SentiBot","Sogou","Spider-Pig","sqlmap","StorygizeBot","Test Certificate Info","TikTokSpider","Timpibot","Turnitin","uipbot","vBSEO","VegeBot","VelenPublicWebCrawler","Web Link Validator","WebCruiser"]}
{"domain":"olympic.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",404]}
{"domain":"animeflv.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"muscache.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"calciomercato.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":17,"stanza_tokens":["ChatGPT-User","PerplexityBot","GPTBot","anthropic-ai","CCBot","Claude-Web","ClaudeBot","cohere-ai","YouBot","Diffbot","OAI-SearchBot","meta-externalagent","Timpibot","Amazonbot","Bytespider","Perplexity-User","*"]}
{"domain":"google.com.cu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"airbus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":139,"stanza_tokens":["AlphaSeoBot","AlphaSeoBot-SA","BLEXBot","008","Alexibot","AlvinetSpider","Antenne Hatena","ApocalXExplorerBot","asterias","BackDoorBot/1.0","BizInformation","Black Hole","BlowFish/1.0","BotALot","BuiltBotTough","Bullseye/1.0","BunnySlippers","Bytespider","Cegbfeieh","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","CopyRightCheck","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","DISCo Pump 3.1","DittoSpyder","dotbot","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","Exabot","ExtractorPro","Flamingo_SearchEngine","Foobot","grapeshot","Harvest/1.5","hloader","httplib","HTTrack","HTTrack 3.0","humanlinks","Igentia","InfoNaviRobot","JennyBot","JikeSpider","Kenjin Spider","LexiBot","libWeb/clsHTTP","LinkextractorPro","LinkScan/8.1a Unix","LinkWalker","lwp-trivial","lwp-trivial/1.34","Mata Hari","Microsoft URL Control - 5.01.4511","Microsoft URL Control - 6.00.8169","MIIxpc","MIIxpc/4.2","Mister PiX","MLBot","moget","moget/2.1","MS Search 4.0 Robot","MS Search 5.0 Robot","Naverbot","NetAnts","NetAttache","NetMechanic","NICErsPRO","Offline Explorer","Openfind","OpenindexSpider","ProPowerBot/2.14","ProWebWalker","psbot","QuepasaCreep","QueryN Metasearch","RepoMonkey","RMA","SightupBot","SiteBot","SiteSnagger","SiteSucker","SpankBot","spanner","Speedy","suggybot","SuperBot","SuperBot/2.6","suzuran","Szukacz/1.4","Teleport","Telesoft","The Intraformant","TheNomad","TightTwatBot"]}
{"domain":"decathlon.es","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"itatiaia.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"prokerala.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":40,"stanza_tokens":["Googlebot-News","Mediapartners-Google","AdsBot-Google","proximic","ia_archiver","008","AhrefsBot","baiduspider","baiduspider-video","baiduspider-image","bytespider","DataForSeoBot","discoverybot","DotBot","Exabot","ichiro","KunatoCrawler","MJ12bot","moget","NaverBot","psbot","PaperLiBot","PetalBot","PiplBot","ScooperBot","SemrushBot","sogou spider","Sosospider","Yandex","YandexImages","Yeti","YoudaoBot","Spinn3r","anthropic-ai","CCBot","Claude-Web","Google-Extended","GPTBot","PerplexityBot","*"]}
{"domain":"ciceksepeti.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["ia_archiver-web.archive.org","ia_archiver","archive.org_bot","*","AdsBot-Google","Google-PageRenderer","OAI-SearchBot","GPTBot","ChatGPT-User","Bytespider","PetalBot","ISSCyberRiskCrawler","img2dataset"]}
{"domain":"kenh14.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"stirileprotv.ro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":3,"stanza_tokens":["*","GPTBot","Bytespider"]}
{"domain":"arconazure.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"queensu.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kemendikdasmen.go.id","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sbisec.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"catho.com.br","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"gayporntube.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"labanquepostale.fr","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"elcinema.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["ImagesiftBot","PetalBot","bingbot","bing","*"]}
{"domain":"google.ps","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"kidoz.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Googlebot","Bingbot","Twitterbot","facebookexternalhit","*"]}
{"domain":"xhonline.blog","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"smbc-card.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Syncbot-Web","*"]}
{"domain":"payhip.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"cyon.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unt.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"correio24horas.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"fdj.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lenta.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["Yandex","*","OAI-SearchBot","GPTBot","ChatGPT-User","ClaudeBot","PerplexityBot","Applebot-Extended","Google-Extended","CCBot"]}
{"domain":"awsdns-39.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bmw.cloud","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsstatic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-54.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"iii.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"parents.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":20,"stanza_tokens":["*","Pinterest","Pinterestbot","ChatGPT-User","OAI-SearchBot","GPTBot","CCBot","news-please","anthropic-ai","ClaudeBot","Claude-Web","cohere-ai","ImagesiftBot","FriendlyCrawler","Quora-Bot","omgilibot","omgili","PerplexityBot","Perplexity-User","Google-Extended"]}
{"domain":"tv3.lt","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"convertkit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"northwesternmutual.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"99acres.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.si","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"lansweeper.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"keenetic.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zerohedge.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"qrcdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"rijksoverheid.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"betking.com.ua","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"iamcook.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"kurashiru.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":6,"stanza_tokens":["*","baiduspider","baiduspider+","Yandex","GPTBot","AmazonAdBot"]}
{"domain":"dailygalaxy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":23,"stanza_tokens":["*","8LEGS","AhrefsBot","AspiegelBot","BLEXBot","Barkrowler","DotBot","MJ12bot","MauiBot","Nimbostratus-Bot","PetalBot","SemrushBot","SeznamBot","Sogou","serpstatbot","trendiction","TextBulkerBot","Amazonbot","Bytespider","CCBot","cohere-ai","FacebookBot","Omgilibot"]}
{"domain":"bancoestado.cl","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"visualsoft.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"portaldasfinancas.gov.pt","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"pw.live","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pptechnology.cc","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cleantalk.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vialotadom.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"qzwxrtyk.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cdnstream1.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"tpd-soe.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"444.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"amazon-dss.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bc.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"funplus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"impression.link","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"thecut.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["ChatGPT-User"],"open":["GPTBot"],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":107,"stanza_tokens":["Mediapartners-Google*","TruliaBot","Twitterbot","*","Amazonbot","Applebot","GPTBot","A6-Indexer","AhrefsBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot-Extended","Baiduspider","betaBot","BLEXBot","botify","brands-bot-logo","Brightbot","Bytespider","calculon","CCBot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","collection@infegy.com","Cotoyogi","Crawl4AI","CrunchBot","deepcrawl","DeepSeekBot","Diffbot","DirBuster","DTS","DuckAssistBot","EasouSpider","FacebookBot","FirecrawlAgent","FriendlyCrawler","GlutenCrawler","Google-CloudVertexBot","Google-Extended","GrapeshotCrawler","iaskspider","ICC-Crawler","ImagesiftBot","img2dataset","JackCrawl","Jetslide","linkapedia_bot","Mediatoolkitbot","MegaIndex","meta-externalagent","meta-externalfetcher","Microsoft BITS","Microsoft URL Control","Microsoft-WebDAV-MiniRedir","MistralAI-User","news-please","Novellum","NTENTbot","Nutch","omgili","Omgilibot","OpenLinkProfiler","Perplexity-User","PerplexityBot","PetalBot","Primalbot","python-requests","Re-Animator","Riddler","SafeDNSBot","SBIntuitionsBot","Scrapy","SearchAtlas","SemanticScholarBot","SemrushBot","SentiBot","Sogou","Spider-Pig","sqlmap","StorygizeBot","Test Certificate Info","TikTokSpider","Timpibot","Turnitin","uipbot","vBSEO"]}
{"domain":"awsdns-37.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wikiloc.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"royalbank.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"oar.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","YandexBot","MJ12bot","SiteimproveBot-Crawler","SiteimproveBot"]}
{"domain":"volkswagen.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Googlebot","Slurp","Bingbot","msnbot","gsa-crawler","*"]}
{"domain":"sport.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","SeznamSocialBot","AI"]}
{"domain":"forocoches.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":12,"stanza_tokens":["Mediapartners-Google*","archive.org_bot","BoardReader","GPTBot","Google-Extended","PerplexityBot","CCBot","FacebookBot","OmgiliBot","anthropic-ai","cohere-ai","*"]}
{"domain":"autohero.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sl.pt","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"mobirise.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"imolive2.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"polovniautomobili.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gazzetta.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rezka.ag","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"kayzen.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"plaync.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[500,404]}
{"domain":"teams-msedge.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nypl.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"bundesregierung.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":44,"stanza_tokens":["AhrefsBot","AlphaBot","Baiduspider","Baiduspider-render","Buck","changedetection","Cliqzbot","Exabot","Flamingo_SearchEngine","Grobbot","JobboerseBot","jobs.de-Robot","linkdexbot","Mail.RU_Bot","Mediatoolkitbot","MegaIndex.ru/2.0","Mega-Index","MegaIndex.ru","MJ12bot","MojeekBot","Pinterestbot","rogerbot","SafeDNSBot","SemrushBot","SemrushBot/2~bl","SemrushBot-BA","SemrushBot-BM","SemrushBot-SA","SEOkicks-Robot","SeznamBot","SMTBot","spbot","trendictionbot","TweetmemeBot","vebidoobot","wonderbot","Wotbox","yacybot","YaK","YandexAntivirus","YandexBot/3.0","YandexImages","YandexMobileBot","Yeti"]}
{"domain":"hentaiheroes.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"knock.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ottadvisors.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"foundationapi.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"gamesradar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","AmazonAdBot","bytespider","mistralai","cohere","ai2bot","youbot","omgili","diffbot","kangaroo","img2dataset","amazonbot","amazon-qbusiness","meta-externalagent","meta-webindexer"]}
{"domain":"mym.fans","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"al.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","ia_archiver","archive.org_bot","heritrix","My-heritrix-crawler","Timpibot","omgili","Bytespider","DuckAssistBot","Applebot-Extended","Google-Extended","Google-CloudVertexBot","GoogleOther","GPTBot","CCBot","Amazonbot","FacebookBot","cohere-ai","cohere-training-data-crawler","Omgilibot","ClaudeBot","Diffbot","PanguBot","MistralAI-User","webz.io","DeepSeekBot","AmazonAdBot","Amzn-SearchBot","Amzn-User","Claude-SearchBot","Claude-User","PetalBot","Meta-ExternalFetcher","Meta-WebIndexer","OAI-SearchBot","ChatGPT-User","PerplexityBot","Perplexity-User"]}
{"domain":"k-img.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-45.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"backtrace.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cloudx.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ladbrokes.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"fanfox.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"boxofficemojo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":8,"stanza_tokens":["*","anthropic-ai","Claude-Web","CCBot","FacebookBot","Google-Extended","GPTBot","PiplBot"]}
{"domain":"davivienda.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":17,"stanza_tokens":["OAI-SearchBot","ChatGPT-User","PerplexityBot","FirecrawlAgent","AndiBot","ExaBot","PhindBot","YouBot","GPTBot","CCBot","Google-Extended","Googlebot","Bingbot","Googlebot-Image","Googlebot-Video","Googlebot-Mobile","*"]}
{"domain":"therange.co.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"soton.ac.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"mediamelon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Googlebot","Bingbot","Twitterbot","facebookexternalhit","*"]}
{"domain":"fanatik.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":12,"stanza_tokens":["*","anthropic-ai","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","GPTBot","OAI-SearchBot","quillbot.com","PerplexityBot","TurnitinBot","AmazonAdBot"]}
{"domain":"site123.me","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"uquiz.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"quay.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"johndeere.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"tgstat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":8,"stanza_tokens":["*","SemrushBot","DataForSeoBot","SemrushBot-SA","dotbot","AhrefsBot","MJ12bot","GPTBot"]}
{"domain":"sciencenews.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":3,"stanza_tokens":["*","GPTBot","ChatGPT-User"]}
{"domain":"mofa.gov.sa","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"frontapp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"naftemporiki.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"autoscout24.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pixnet.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"azionedge.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"felissimo.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Amazonbot","*"]}
{"domain":"flashscore.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"alchemy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"egais.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"manhuaus.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"zg-api.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[502,404]}
{"domain":"internshala.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":["ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot","anthropic-ai"],"open":["Claude-User"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":44,"stanza_tokens":["*","Baiduspider","Baiduspider-image","Baiduspider-video","Baiduspider-news","Baiduspider-favo","Baiduspider-cpro","Baiduspider-ads","MSNBot","MSNBot-Media","AdIdxBot","GPTBot","OAI-SearchBot","Google-Extended","meta-externalagent","Amazonbot","GoogleOther","Anthropic-AI","Claude-Web","ClaudeBot","Claude-SearchBot","PerplexityBot","Cohere","cohere-ai","Applebot-Extended","Google-CloudVertexBot","DotBot","spbot","AhrefsBot","CCBot","Omgilibot","Omgili","FacebookBot","Bytespider","Bytedance","Diffbot","Youbot","FriendlyCrawler","img2dataset","ChatGPT-User","Perplexity-User","Claude-User","meta-webindexer","facebookexternalhit"]}
{"domain":"marvel.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"adam4adam.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"alberta.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"turboscribe.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"l3net.works","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","DEPTH_ZERO_SELF_SIGNED_CERT"]}
{"domain":"nissen.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Mediapartners-Google","bingbot","meta-externalagent"]}
{"domain":"hentaihaven.xxx","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mgstage.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"popin.cc","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"azet.sk","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"oikotie.fi","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ut.ac.id","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tportal.hr","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"croma.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[403,200]}
{"domain":"gala.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":237,"stanza_tokens":["*","ChatGPT-User","Claude-SearchBot","Claude-Web","OAI-SearchBot","Claude-User","ChatGPT Agent","GoogleAgent-Mariner","Google-NotebookLM","Google-CloudVertexBot","MistralAI-User","Gemini-Deep-Research","CloudVertexBot","anthropic-ai","Bytespider","CCBot","ClaudeBot","cohere-ai","GPTBot","Google-Extended","Alexibot","AlvinetSpider","AmiSoftware","Antenne Hatena","ApifyBot","ApifyWebsiteContentCrawler","ApocalXExplorerBot","Argus","Ask n read","asknread.com","asterias","BlowFish/1.0","BotALot","Brightbot","BuiltBotTough","Bullseye/1.0","BunnySlippers","Cegbfeieh","CheeseBot","ConveraCrawler","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","cydralspider","Diffbot","Diffbot-User","DISCo Pump 3.1","DittoSpyder","EroCrawler","eureka","ExaBot","Explore","Fetch","FirecrawlAgent","Flamingo_SearchEngine","Foobot","gammaSpider","grub-client","hloader","httplib","humanlinks","ia_archiver","ia_archiver-web.archive.org","indexer","InfoNaviRobot","infoseek","JennyBot","Jetbot","JikeSpider","k2spider","Kenjin Spider","larbin","LexiBot","libWeb/clsHTTP","libwww","linko","LinkScan/8.1a Unix","LinkWalker","lwp-trivial","lwp-trivial/1.34","Mata Hari","Mister PiX","MLBot","moget","moget/2.1","Naverbot","NetAttache","Newzbin","NICErsPRO","NPBot","ObjectsSearch","omgili","omgilibot","Openfind","OpenindexSpider","Pimptrain","ProPowerBot/2.14","ProWebWalker","psbot","QuepasaCreep"]}
{"domain":"unirioja.es","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":["ENOTFOUND",429,429]}
{"domain":"ccc.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mts-nn.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","TimeoutError"]}
{"domain":"wsj.net","fetch_ok":false,"usable":false,"reason":"other","status":401,"error":null,"attempts":[401,401]}
{"domain":"theyarehuge.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*\r\rDisallow: /player/float.php\rDisallow: /player/html.php\rDisallow: /redirect_cs.php\rDisallow: /*?action=*$\rDisallow: /*mode=async*$\rDisallow: /*function=get_block*$\rDisallow: /404.php\rDisallow: /logout/\rDisallow: /email/\rDisallow: /payments/\rDisallow: /login/"]}
{"domain":"case.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yutex.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"coinmaster.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"windsurf.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tu-dresden.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zoom.com.cn","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":[503,503,503]}
{"domain":"awsdns-43.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"yzwtz.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"mediafax.ro","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"glamour.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","FacebookExternalHit","Twitterbot","Pinterestbot","LinkedInBot","TikTokSpider","Storebot-Google","Google-InspectionTool","AmazonAdBot","Google-Extended","Google-CloudVertexBot","GoogleOther","Applebot-Extended","Amazonbot","meta-webindexer","meta-externalagent","meta-externalfetcher","ClaudeBot","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","cohere-training-data-crawler","cohere-ai","CCBot","PanguBot","PetalBot","Bytespider","Diffbot","DuckAssistBot","MistralAI-User","Timpibot","Webzio","Webzio-Extended","archive.org_bot","ia_archiver","ia_archiver-web.archive.org","heritrix"]}
{"domain":"inquirer.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":4,"stanza_tokens":["*","Petalbot","GPTBot","facebookexternalhit"]}
{"domain":"islonline.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"monetixads.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AdsBot-Google-Mobile","AdsBot-Google","PetalBot","dotbot","AhrefsBot"]}
{"domain":"clickbank.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"purepeople.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["AmazonAdBot","*","Arquivo-web-crawler","AI2Bot","ChatGLM","ChatGLM-spider","cohere-training-data-crawler","Diffbot","OmigiliBot","PetalBot","YouBot"]}
{"domain":"fotomac.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pornpics.de","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ECONNRESET"]}
{"domain":"porngameshub.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mailspike.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"awsdns-60.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"anythinktech.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNREFUSED"]}
{"domain":"jetpunk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"gauthmath.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"c-span.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"musicdel.ir","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot-Image"]}
{"domain":"record.pt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":7,"stanza_tokens":["OpenAI-Crawler","Googlebot-Bard","GPTBot","ChatGPT-User","Google-Extended","CCBot","*"]}
{"domain":"stitcher.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"mxpnl.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mahaonline.gov.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"disquscdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"theweek.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","AmazonAdBot","bytespider","mistralai","cohere","ai2bot","youbot","omgili","diffbot","kangaroo","img2dataset","amazonbot","amazon-qbusiness","meta-externalagent","meta-webindexer"]}
{"domain":"lithium.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"domclick.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Yandex","Googlebot","*"]}
{"domain":"ts.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","atlassian-bot"]}
{"domain":"skai.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"auappservice.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"bethesda.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zerozero.pt","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"awsdns-10.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"snai.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"friv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mtb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nxp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Baiduspider"]}
{"domain":"novelupdates.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"53.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"aastocks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"dataannotation.tech","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ngs.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"topky.sk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"mi-fds.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vix.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thescore.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"immoweb.be","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"almalinux.org","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[404,200]}
{"domain":"fantia.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uni-kiel.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["bingbot","Googlebot","YandexBot","SemanticScholarBot","Applebot","Baiduspider","DuckDuckBot","PetalBot","facebookexternalhit","Twitterbot","*"]}
{"domain":"daily.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AmazonAdBot","*"]}
{"domain":"dropboxstatic.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENETUNREACH",404]}
{"domain":"saksfifthavenue.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"ancestry.com.au","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","proximic","sogou spider","Google-Extended"]}
{"domain":"manual.canon","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"awsdns-35.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"gn-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pgammedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Googlebot","Bingbot","Twitterbot","facebookexternalhit","*"]}
{"domain":"whatnot.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"globalsigncdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"myshn.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"beatport.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"roeyecdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ENOTFOUND"]}
{"domain":"ipsosinteractive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":2,"stanza_tokens":["PerplexityBot","*"]}
{"domain":"alaskaair.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"kunlunhuf.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bamboohr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","GPTBot","ClaudeBot","Google-Extended","Anthropic-AI","Bytespider","CCBot","Applebot-Extended","Meta-ExternalAgent","OAI-SearchBot","PerplexityBot"]}
{"domain":"digitaljournal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"imss.gob.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gamestar.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":38,"stanza_tokens":["AI2Bot","ChatGLM","ChatGLM-Spider","cohere-training-data-crawler","Diffbot","OmigiliBot","PetalBot","YouBot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","GPTBot","Meta-ExternalAgent","OAI-SearchBot","Perplexity-User","PerplexityBot","larbin","strucr","psbot","Python-urllib","lwp-trivial/1.34","lwp-trivial","lwp-request","MagpieRSS","Custo","ichiro","Sensis Web Crawler","IRLbot","Exabot","CFNetwork","TencentTraveler","Yandex","AhrefsBot","*"]}
{"domain":"fsf.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":47,"stanza_tokens":["*","MJ12bot","DataForSeoBot","BLEXBot","AhrefsBot","barkrowler","Screaming Frog SEO Spider","ZoomBot","magpie-crawler","DotBot","rogerbot","SemrushBot","SiteAuditBot","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SplitSignalBot","SemrushBot-OCOB","JamesBOT","Oncrawl","AwarioRssBot","AwarioSmartBot","AwarioBot","serpstatbot","netEstate NE Crawler","panscient.com","AcademicBotRTU","TurnitinBot","CheckMarkNetwork","SEOkicks","Timpibot","Seobility","BitSightBot","linkfluence","adbeat_bot","BrandVerity","peer39_crawler","PiplBot","Scrapy","BUbiNG","undici","YisouSpider","TerraCotta","HaloBot","SERankingBacklinksBot","AffsignalCrawler","RankzonBot"]}
{"domain":"eroprofile.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Amazonbot","AmazonAdBot"]}
{"domain":"district.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","AdsBot-Google","AdsBot-Google-Mobile-Apps","Mediapartners-Google"]}
{"domain":"ultipro.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"tapsell.ir","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"extremereach.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tiktoklb.eu","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"acuityscheduling.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vatanbilgisayar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AhrefsBot","SemrushBot","Yandex","YandexBot","GPTBot"]}
{"domain":"onlinetestpad.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"avon.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"shaw.ca","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"assets-yammer.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cdnhwcljk104.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"jito.wtf","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jav.guru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cashify.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"noticiasaominuto.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":38,"stanza_tokens":["Twitterbot","libwww-perl","*","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Diffbot","DuckAssistBot","FriendlyCrawler","GPTBot","iaskspider/2.0","ICC-Crawler","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","omgili","omgilibot","PanguBot","PetalBot","Scrapy","SemrushBot","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot"]}
{"domain":"govee.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mangooutlet.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"losandes.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"washingtonexaminer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["*","Bytespider","YandexBot","Spinn3r","Yandex","Baidu","Mail.RU_Bot","CCBot","omgilibot","omgili","Sogou","Exabot","Coccocbot"]}
{"domain":"deviantart.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"argentina.gob.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"porntrex.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"tubegalore.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"mathplayground.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","ia_archiver","*"]}
{"domain":"google.com.om","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"kinogo.ec","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["Yandex","*","BLEXBot","MJ12bot","SemrushBot","PetalBot","CriteoBot","proximic","grapeshot","ia_archiver"]}
{"domain":"ncl.ac.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"msport.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"outdooractive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":651,"stanza_tokens":["*","anthropic-ai","Google-Extended","GPTBot","Applebot-Extended","cohere-ai","Diffbot","01h4x.com","360Spider","404checker","404enemy","80legs","ADmantX","AIBOT","ALittle Client","ASPSeek","Abonti","Aboundex","Aboundexbot","Acunetix","AdsTxtCrawlerTP","AfD-Verbotsverfahren","AhrefsBot","AiHitBot","Aipbot","Alexibot","AllSubmitter","Alligator","AlphaBot","Anarchie","Anarchy","Anarchy99","Ankit","Anthill","Apexoo","Aspiegel","Asterias","Atomseobot","Attach","AwarioRssBot","AwarioSmartBot","BBBike","BDCbot","BDFetch","BLEXBot","BackDoorBot","BackStreet","BackWeb","Backlink-Ceck","BacklinkCrawler","Badass","Bandit","Barkrowler","BatchFTP","Battleztar Bazinga","BetaBot","Bigfoot","Bitacle","Black Hole","BlackWidow","Blackboard","Blow","BlowFish","Boardreader","Bolt","BotALot","Brandprotect","Brandwatch","Buck","Buddy","BuiltBotTough","BuiltWith","Bullseye","BunnySlippers","BuzzSumo","Bytespider","CATExplorador","CCBot","CODE87","CSHttp","Calculon","CazoodleBot","Cegbfeieh","CensysInspect","CheTeam","CheeseBot","CherryPicker","ChinaClaw","Chlooe","Citoid","Claritybot","Cliqzbot","Cloud mapping","Cocolyzebot","Cogentbot","Collector","Copier","CopyRightCheck","Copyscape","Cosmos"]}
{"domain":"skole.hr","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"allporncomic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"dmv.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"miniclip.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adikteev.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.co.ao","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"loudecho.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"temple.edu","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"soha.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"booklive.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"asianet.co.th","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ETIMEDOUT"]}
{"domain":"nld.com.vn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"awsdns-46.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"slatic.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"poly.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"sakshi.com","fetch_ok":false,"usable":false,"reason":"other","status":405,"error":null,"attempts":[405,405]}
{"domain":"elfsight.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sohu01.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"library.link","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"alibabachengdun.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"cernet.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNRESET"]}
{"domain":"websitedemos.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mangago.zone","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"elbalad.news","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","Googlebot-Mobile","AdsBot-Google-Mobile","bingbot-mobile","Mediapartners-Google","APIs-Google","AdsBot-Google","Googlebot-Image"]}
{"domain":"engagelab.cc","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xxbrits.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mangakatana.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["ia_archiver"]}
{"domain":"loilonote.app","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"mangaread.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bezeq.co.il","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"casa.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eapteka.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot","Yandex"]}
{"domain":"palcloset.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ClaudeBot","PerplexityBot","Google-Extended"]}
{"domain":"tukrd.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"icicidirect.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":139,"stanza_tokens":["*","OmniExplorer_Bot","FreeFind","BecomeBot","Nutch","Jetbot/1.0","Jetbot","WebVac","Stanford","naver","dumbot","Hatena Antenna","grub-client","grub","WebZip","larbin","b2w/0.1","Copernic","psbot","Python-urllib","NetMechanic","URL_Spider_Pro","CherryPicker","EmailCollector","EmailSiphon","WebBandit","EmailWolf","ExtractorPro","CopyRightCheck","Crescent","SiteSnagger","ProWebWalker","CheeseBot","LNSpiderguy","Alexibot","Teleport","TeleportPro","Stanford Comp Sci","MIIxpc","Telesoft","Website Quester","moget/2.1","WebZip/4.0","WebStripper","WebSauger","WebCopier","NetAnts","Mister PiX","WebAuto","TheNomad","WWW-Collector-E","RMA","libWeb/clsHTTP","asterias","httplib","turingos","spanner","InfoNaviRobot","Harvest/1.5","Bullseye/1.0","Crescent Internet ToolPak HTTP OLE Control v.1.0","CherryPickerSE/1.0","CherryPickerElite/1.0","WebBandit/3.50","NICErsPRO","Microsoft URL Control - 5.01.4511","DittoSpyder","Foobot","SpankBot","BotALot","lwp-trivial/1.34","lwp-trivial","BunnySlippers","Microsoft URL Control - 6.00.8169","URLy Warning","Wget/1.6","Wget/1.5.3","Wget","LinkWalker","cosmos","moget","hloader","humanlinks","LinkextractorPro","Offline Explorer","Mata Hari","LexiBot","Web Image Collector","The Intraformant","True_Robot/1.0","True_Robot","BlowFish/1.0","JennyBot","MIIxpc/4.2","BuiltBotTough","ProPowerBot/2.14","BackDoorBot/1.0","toCrawl/UrlDispatcher","WebEnhancer","suzuran"]}
{"domain":"thepure.app","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"anzu-us.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"scarabresearch.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ENOTFOUND"]}
{"domain":"perfectgirls.xxx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"onlinekhabar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"my-personaltrainer.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":19,"stanza_tokens":["*","Googlebot","Mediapartners-Google","Googlebot-Image","Claude-Web","anthropic-ai","cohere-ai","PerplexityBot","seekr","Meltwater","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"tiava.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"depop.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"gendai.media","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":16,"stanza_tokens":["*","carbon-umbrella-bot/1.0","CCBot","GPTBot","Applebot-Extended","Google-Extended","Meta-ExternalAgent","ClaudeBot","anthropic-ai","Claude-Web","cohere-ai","omgili","Timpibot","Webzio-Extended","Bytespider","ICC-Crawler"]}
{"domain":"cntv.cn","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"honeytoon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rolimons.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":23,"stanza_tokens":["adidxbot","AdsBot-Google","AhrefsBot","AliyunSecBot","AmazonAdBot","baiduspider","baiduspider-mobile","bingbot","BLEXBot","Googlebot","Googlebot-Mobile","GumGum","GumGum-Bot","Mediapartners-Google","MJ12bot","msnbot","PetalBot","slurp","weborama","weborama-fetcher","Yandex","Bytespider","*"]}
{"domain":"obilet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["facebookexternalhit","GPTBot","Google-Extended","PerplexityBot","*"]}
{"domain":"oceanofpdf.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["AhrefsBot","proximic","TTD-Content","Mediapartners-Google","*"]}
{"domain":"multiscreensite.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"qcloudcdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","attempts":["ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"erothots.co","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":[429,429,429]}
{"domain":"google.am","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"nabu.casa","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNRESET"]}
{"domain":"ad-m.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ruidongcloud.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"modal.host","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-16.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"minecraft-inside.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"dagospia.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"bcb.gov.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Googlebot","Googlebot-Image","Mediapartners-Google","Adsbot-Google","AdsBot-Google-Mobile-Apps","Googlebot-News","Googlebot-Video","AdsBot-Google-Mobile"]}
{"domain":"polsatnews.pl","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"iot-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cbse.gov.in","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"google.com.pg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"therapynotes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fitgirl-repacks.site","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"starwars.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":5,"stanza_tokens":["*","dotbot","rogerbot","GPTBot","Google-Extended"]}
{"domain":"ygosu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":15,"stanza_tokens":["Bytespider","AhrefsBot","SemrushBot","GPTBot","ClaudeBot","CCBot","Google-Extended","PerplexityBot","Amazonbot","Applebot-Extended","PetalBot","YandexBot","MJ12bot","DotBot","*"]}
{"domain":"tokubai.co.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pocketoption.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AhrefsBot","StackRambler","*"]}
{"domain":"ccb.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_UNSAFE_LEGACY_RENEGOTIATION_DISABLED","attempts":["ERR_SSL_UNSAFE_LEGACY_RENEGOTIATION_DISABLED","ERR_SSL_UNSAFE_LEGACY_RENEGOTIATION_DISABLED"]}
{"domain":"csfloat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ikman.lk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["Mediapartners-Google","Adsbot-Google","Googlebot-Image","*","msnbot","CCBot","Baiduspider","Sogou","BLEXBot"]}
{"domain":"ably.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ouedkniss.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sbicard.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","ETIMEDOUT"]}
{"domain":"we-stats.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"news-cdn.site","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"goodnotes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","GPTBot","Google-Extended","ClaudeBot","PerplexityBot","Applebot-Extended"]}
{"domain":"google.bi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"post-gazette.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lachainemeteo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":143,"stanza_tokens":["AmazonAdBot","admantx","outbrain","criteo","Scope3","thetradedesk","ozone","ApocalXExplorerBot","asterias","BackDoorBot/1.0","Black Hole","BlowFish/1.0","BotALot","BuiltBotTough","Bullseye/1.0","BunnySlippers","Cegbfeieh","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","cosmos","Crescent","DIIbot","DISCo Pump 3.1","Download Ninja","downloadexpress","eCatch","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","Explore","ExtractorPro","Flamingo_SearchEngine","Foobot","gammaSpider","Harvest/1.5","hloader","humanlinks","indexer","InfoNaviRobot","JennyBot","Jetbot","k2spider","Kenjin Spider","LexiBot","LinkextractorPro","Mata Hari","NetAnts","NPBot","ObjectsSearch","Offline Explorer","Pimptrain","ProPowerBot/2.14","ProWebWalker","QuepasaCreep","QueryN Metasearch","RepoMonkey","RMA","Roverbot","SightupBot","SiteSnagger","SiteSucker","SpankBot","spanner","suggybot","SuperBot","Teleport","TeleportPro","Telesoft","vsw","wapspider","WebBandit","WebCopier","AgentCore-Browser","Bloodhound","Cotoyogi","CotoyogiBot","Datenbank Crawler","DownloadNinja","ICC-Crawler","toCrawl/UrlDispatcher","WARDBot","WebSauger","website extractor","Website Quester","Webster Pro","WebStripper/2.02","WebZinger","WebZip","WebZip/4.0","WWW-Collector-E","YRSPider","ZanistaBot","Zealbot","Zeus 32297 Webster Pro V2.9 Win32","Zookabot","ZyBORG","Sogou web spider"]}
{"domain":"babepedia.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"salesforce-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ripley.cl","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tiempo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"widen.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"kuiniuca.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"sofifa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"amebaownd.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"homeadvisor.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"wurflcloud.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"vivint.ai","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ECONNRESET"]}
{"domain":"gaymaletube.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"championat.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Googlebot","Yandex","*"]}
{"domain":"bobaedream.co.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","grapeshot","Amazonbot"]}
{"domain":"photoshelter.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":9,"stanza_tokens":["*","MJ12bot","Twitterbot","PetalBot","Riddler","Baiduspider","AhrefsBot","Amazonbot","ClaudeBot"]}
{"domain":"ledger.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","GPTBot","ClaudeBot","PerplexityBot","Google-Extended"]}
{"domain":"super.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","SeznamSocialBot","AI"]}
{"domain":"blikk.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"annas-archive.gl","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"clutch.co","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"wbs.legal","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-00.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.dz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"doz.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":28,"stanza_tokens":["*","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","wget","grub-client","k2spider","NPBot","WebReaper"]}
{"domain":"awsdns-31.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"a2dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"suspilne.media","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"animepahe.pw","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"horoscope.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","grapeshot"]}
{"domain":"awsdns-30.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"t-static.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"intersport.fr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"medicinenet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":9,"stanza_tokens":["stress-agent","Googlebot","GPTBot","CCBot","ClaudeBot","Ultraseek","ChatGPT-User","Doximity","*"]}
{"domain":"xvideos3.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"1adr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"deine-tierwelt.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":46,"stanza_tokens":["*","Adsbot-Google","AhrefsSiteAudit","Amazonbot","anthropic-ai","Applebot","Baiduspider","Bingbot","Browsershots","CCBot","DuckDuckBot","eBay Relevance Ad Crawler","eBay Relevance Ad Crawler powered by contentDetection (www.mindup.de)","Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)","Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)","facebookcatalog/1.0","facebookexternalhit","facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)","facebookexternalhit/1.1","facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)","Feedfetcher-Google","Googlebot","Google-Extended","Googlebot-Image","Googlebot-Mobile","Googlebot-News","Googlebot-Video","GPTBot","grapeshot","GumGum","Heritrix","ia_archiver","LinkedInBot","MediaPartners-Google","Mediapartners (Googlebot)","msnbot","PerplexityBot","Pinterestbot","Slackbot","Slurp","Sogou","TEOMA","TermLabs","Twitterbot","YandexBot","Bytespider"]}
{"domain":"flashscore.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"foreca.fi","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"neu.edu.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"arabam.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"martianinc.co","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bytegslb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"healthfusionclaims.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"paychex.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"milffox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dot-st.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"photonengine.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"swatch.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kinsta.cloud","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"mufg.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"123moviesfree.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fortnite.gg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"gnavi.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"glosbe.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":10,"stanza_tokens":["*","008","proximic","ia_archiver","CCBot","voltron","TurnitinBot","Paracrawl","GPTBot","Bytespider"]}
{"domain":"jang.com.pk","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nap.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":16,"stanza_tokens":["*","Baiduspider","Googlebot","bingbot","msnbot","Charlotte","GPTBot","ChatGPT-User","Google-Extended","CCBot","PerplexityBot","anthropic-ai","Claude-Web","ClaudeBot","ChatGLM-Spider","Bytespider"]}
{"domain":"21vek.by","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"071kk.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"anrdoezrs.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"leroymerlin.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["Googlebot","AdsBot-Google","AdsBot-Google-mobile","Googlebot-image","OAI-SearchBot","*"]}
{"domain":"google.gg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"emojidb.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"rakuten-bank.co.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"marketscreener.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"zblogcn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"x-video.tube","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thehentai.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"financialcontent.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"ethereum.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"buzzvil.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bazos.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Mediapartners-Google","trovitBot","pricebot","magpie-crawler","SemrushBot","trendictionbot","BLEXBot","Riddler"]}
{"domain":"multporn.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Googlebot","Googlebot-Image","Yandex","*"]}
{"domain":"ncaa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":27,"stanza_tokens":["*","AI2Bot","Ai2Bot-Dolma","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","cohere-ai","cohere-training-data-crawler","Diffbot","FacebookBot","Google-Extended","GPTBot","Kangaroo Bot","Meta-ExternalAgent","meta-externalagent","Meta-ExternalFetcher","meta-externalfetcher","omgili","omgilibot","Timpibot","Webzio-Extended","magpie-crawler","Scrapy"]}
{"domain":"paytm.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":[429,429,429]}
{"domain":"cri.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fly.dev","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"digilocker.gov.in","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"bloomberglaw.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Googlebot","Google-InspectionTool","Bingbot","BingPreview","*"]}
{"domain":"rbstsystems.live","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.ci","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"x-x-x.tube","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","ClaudeBot","Google-Extended","Bytespider","Amazonbot","PerplexityBot","CCBot","anthropic-ai","cohere-ai"]}
{"domain":"bibliaonline.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tudogostoso.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["AmazonAdBot","*","AI2Bot","ChatGLM","ChatGLM-spider","cohere-training-data-crawler","Diffbot","OmigiliBot","PetalBot","YouBot"]}
{"domain":"radios.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"newsnow.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":7,"stanza_tokens":["AhrefsBot","SemrushBot","anthropic-ai","GPTBot","PetalBot","AmazonAdBot","*"]}
{"domain":"getjobber.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":117,"stanza_tokens":["OAI-SearchBot","ChatGPT-User","ChatGPT-User/2.0","GPTBot","ClaudeBot","anthropic-ai","Claude-Web","Gemini-Deep-Research","Google-Extended","Applebot-Extended","Meta-ExternalAgent","FacebookBot","PerplexityBot","Bingbot","cohere-ai","CCBot","YouBot","BraveBot","Googlebot","Googlebot-Image","Slurp","DuckDuckBot","Yandex","Applebot","Twitterbot","LinkedInBot","Pinterest","Slackbot","TelegramBot","WhatsApp","Discordbot","*","HTTrack","WebCopier","WebZIP","WebStripper","SiteSnagger","Teleport","TeleportPro","WebCapture","Offline Explorer","Wget","curl","DotBot","MJ12bot","BLEXBot","DataForSeoBot","SeznamBot","PetalBot","Bytespider","serpstatbot","SEOkicks","SEOkicks-Robot","sistrix","JEOD","libwww-perl","Python-urllib","Scrapy","Go-http-client","Java","Jakarta","Mechanize","Nutch","MegaIndex","ltx71","BLP_bbot","CompSpyBot","ChinaClaw","Harvest","EmailCollector","EmailSiphon","EmailWolf","ExtractorPro","Xenu","Zeus","UptimeBot","UptimeRobot","StatusCake","Pingdom","Site24x7","Baiduspider","Sogou","YisouSpider","360Spider","Exabot","GrapeshotCrawler","Proximic","SafeDNSBot","Cliqzbot","magpie-crawler","Ezooms","ia_archiver","archive.org_bot","Wayback","Nikto","sqlmap","Nessus","OpenVAS","Acunetix","Nmap"]}
{"domain":"gestiondecuenta.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"voodoo-gaming.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ns14.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"towardsdatascience.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":20,"stanza_tokens":["*","GPTBot","ChatGPT-User","CCBot","anthropic-ai","ClaudeBot","Claude-Web","FacebookBot","PerplexityBot","Cohere-ai","Seekr","Amazonbot","Bytespider","BLEXBot","ImagesiftBot","OAI-SearchBot","Applebot","YandexBot","GPTbot","meta-externalagent"]}
{"domain":"cdn-net.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,404]}
{"domain":"aditude.cloud","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"opendsp.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"otieu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"car.gr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sexkomix2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pmi.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tukif.porn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["archive.org_bot","ia_archiver","*"]}
{"domain":"google.mu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"elitebabes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nptel.ac.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bluetrafficstream.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"modland.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tvp.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dinotube.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"revistaoeste.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AmazonAdBot","*"]}
{"domain":"google.com.ag","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"autotrader.co.za","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":["TimeoutError",503,503]}
{"domain":"metoffice.gov.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Swiftbot","*","proximic"]}
{"domain":"stoiximan.gr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"cylance.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"yadi.sk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ss.lv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"alfasense.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nordpass.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["ia_archiver","archive.org_bot","ia_archiver-web.archive.org","*"]}
{"domain":"ingresso.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Mediapartners-Google"]}
{"domain":"etoro.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":59,"stanza_tokens":["Googlebot","Googlebot-News","Googlebot-Image","Googlebot-Video","GoogleOther","*","BaiduSpider","CompSpyBot","Curious George","CybEye.com","DoCoMo","ExB Language Crawler","Ezooms","Flamingo_SearchEngine","Genieo","Genio","LWNutch","LexxeBot","OpenWebIndex","RediffNewsBot","SEOENGWorldBot","Scanmine","ShopWiki","ShowyouBot","Sosospider","WocBot","YoudaoBot","daumoa","gsa-crawler","libcrawl","linkdex","magpie-crawler","repparser","rogerbot","sindice-site-manager","sogou spider","sogou","woriobot","yacybot","yolinkBot","Nutch","Qwantify","trendkite-akashic-crawler","JamesBOT","BUbiNG","SearchmetricsBot","arquivo-web-crawler","www.analyticsseo.com/crawler","HubSpot","dotbot","Cliqzbot","spbot","NTENTbot","SocialRankIOBot","Mediatoolkitbot","Companybook Crawler","Domain Re-Animator Bot","Mail.RU_Bot","heritrix"]}
{"domain":"maurices.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"tupi.fm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["Googlebot-News","Googlebot","*","Googlebot-Image","AmazonAdBot"]}
{"domain":"google.cat","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"adultwork.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"n2ch.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ingentaconnect.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["GoogleBot","*"]}
{"domain":"destatis.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bambulab.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"de17a.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"impactradius-event.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ECONNRESET",403]}
{"domain":"misa.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"porno666.nl","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"wnp.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":318,"stanza_tokens":["*","Bingbot","Yandex","008","010","360Spider","80legs","Aboundex","accelobot","Add\\ Catalog","AhrefsBot","aiHitBot","Alexibot","Aqua_Products","AskJeeves","asterias","awcheckBot","b2w/0.1","BackDoorBot/1.0","BacklinkCrawler","Baiduspider","Barkrowler","BecomeBot","BLEXBot","BlowFish/1.0","Bookmark search tool","BotALot","brandwatch.net","BrightEdge Crawler","BuiltBotTough","Bullseye/1.0","BunnySlippers","Butterfly","CatchBot","Charlotte","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","CLIPish","Cliqzbot","Cocolyzebot","COMODO","Comodo-Certificates-Spider","CompSpyBot","Copernic","CopyRightCheck","cosmos","crawler","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","Curious","curl","DataForSeoBot","dataprovider\\.com","DinoPing","discoverybot","DittoSpyder","DomainCrawler","DomainStatsBot","dotbot","dotnetdotcom","Dow\\ Jones\\ Searchbot","dumbot","EasouSpider","EmailCollector","EmailSiphon","EmailWolf","Enterprise_Search","Enterprise_Search/1.0","EroCrawler","es","Exabot","ExtractorPro","EzineArticlesLinkScanner","Ezooms","FairAd Client","Flaming AttackBot","Foobot","FreeFind","FTRF\\:\\ Friendly","Gaisbot","GetRight/4.2","gigabot","grub","grub-client","Harvest/1.5","Hatena Antenna","hloader","http://www.SearchEngineWorld.com bot","http://www.WebmasterWorld.com bot","HTTP_Request","HTTP_Request2","httplib","humanlinks","hypestat","ia_archiver","ia_archiver/1.6","Indy\\ Library","InfoNaviRobot"]}
{"domain":"apkmodjoy.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vasdgame.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fikfap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thothd.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"personal.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AdsBot-Google"]}
{"domain":"awsdns-08.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"arttrk.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"nsctotal.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"videocelebs.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pornobolt.in","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"meteomedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","AmazonAdBot","Hootsuite","HootsuiteBot","OgScrper","OgScraper","Hootsuite-Authoring"]}
{"domain":"bustimes.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":32,"stanza_tokens":["*","ImagesiftBot","AdsBot-Google","Mediapartners-Google","Google-Display-Ads-Bot","AmazonAdBot","ias_crawler","ias_wombles","proximic","TTD-Content","Quantcastbot","CriteoBot/0.1","InteractiveAdvertisingBureau","Leikibot","GrapeshotCrawler","adidxbot","Yahoo Ad Monitoring","peer39_crawler","Connatix/FeedCrawler","Connatix/WebCrawler","bidswitchbot","Verity","GumGumAdsTxtCrawler/1.0","IAB-Tech-Lab","TheCoolCompany-Crawler","SPM-AdsTxt-Verifier","OpenXAdsTxtCrawler","PubMatic Crawler Bot","Seedtag-Node-Request","adstxt/1.0","Jounce","Kargo Ads.txt Crawler"]}
{"domain":"p-bandai.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","dumbot"]}
{"domain":"psiphon.ca","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"dinamalar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["ChatGPT-User"],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":5,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","CCBot"]}
{"domain":"ngl.link","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"linkplay.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":30,"stanza_tokens":["AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","DuckAssistBot","FacebookBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","img2dataset","Meta-ExternalAgent","MyCentralAIScraperBot","omgili","omgilibot","Quora-Bot","TikTokSpider","YouBot","AdsBot-Google","AdsBot-Google-Mobile","AdsBot-Google-Mobile-Apps","*"]}
{"domain":"livechatenterprise.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"google.com.lb","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"pla3.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"litmarket.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":14,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","ClaudeBot","Claude-Web","FacebookBot","CCBot","SemrushBot","AhrefsBot","MJ12bot","DotBot","PetalBot","BLEXBot"]}
{"domain":"esocial.gov.br","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"leroymerlin.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["Yandex","YandexTurbo","YandexBot","*","008","msnbot","bingbot","Pinterest"]}
{"domain":"mandmdirect.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cash.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kinogo.online","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["Yandex","*","BLEXBot","MJ12bot","SemrushBot","PetalBot","CriteoBot","proximic","grapeshot","ia_archiver"]}
{"domain":"chichi-pui.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["ImagesiftBot","ias-va","ias-or","ias-ie","*"]}
{"domain":"livesport.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"kaliscan.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.ad","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"kozow.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"tu-berlin.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":136,"stanza_tokens":["*","Baiduspider","AddSearchBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","amazon-kendra","Amazonbot","AmazonBuyForMe","Amzn-SearchBot","Amzn-User","Andibot","Anomura","anthropic-ai","Applebot","Applebot-Extended","atlassian-bot","Awario","AzureAI-SearchBot","bedrockbot","bigsur.ai","Bravebot","Brightbot 1.0","BuddyBot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT Agent","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","Cloudflare-AutoRAG","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Crawlspace","Datenbank Crawler","DeepSeekBot","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","FacebookBot","facebookexternalhit","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","Google-Firebase","Google-NotebookLM","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iAskBot","iaskspider","iaskspider/2.0","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","img2dataset","ISSCyberRiskCrawler","kagi-fetcher","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","LAIONDownloader","LCC","LinerBot","Linguee Bot","LinkupBot","Manus-User","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","meta-webindexer","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NotebookLM","NovaAct","OAI-SearchBot","omgili","omgilibot","OpenAI","Operator"]}
{"domain":"storiesig.info","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"shopier.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"njuskalo.hr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":11,"stanza_tokens":["*","Googlebot","Googlebot-Image","MediaPartners-Google","AdsBot-Google-Mobile","AdsBot-Google","Facebot","facebookexternalhit","OAI-SearchBot","ChatGPT-User","GPTBot"]}
{"domain":"maturetube.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"lpsg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eitaa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"newsday.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":3,"stanza_tokens":["*","GPTBot","CCBot"]}
{"domain":"christies.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"mgm.gov.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["meta-externalagent","Amazonbot","*"]}
{"domain":"toongod.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"7-11.com.tw","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"leggo.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":21,"stanza_tokens":["*","GPTBot","Google-Extended","anthropic-ai","CCBot","Omgilibot","cohere-ai","ChatGPT-User","OAI-SearchBot","PerplexityBot","Amazonbot","YouBot","Claude-Web","Claudebot","Claude-User","Claude-SearchBot","Diffbot","Bytespider","ImagesiftBot","Meltwater","Seekr"]}
{"domain":"auntmia.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"premierbet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hu.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"openrouter.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yad2.co.il","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fout.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"4dvertible.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"senkodns.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND",403]}
{"domain":"javhdporn.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":40,"stanza_tokens":["bingbot","coccocbot-web","AhrefsBot","YandexBot","*","Googlebot","Twitterbot","Amazonbot","Anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","Barkrowler","BLEXBot","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","Cohere-ai","DataForSeoBot","DotBot","FacebookBot","GPTBot","ImagesiftBot","Magpie-crawler","MJ12bot","Omgili","Omgilibot","Peer39_crawler","Peer39_crawler/1.0","PerplexityBot","YouBot","YaK","Y!J-ASR","SeznamBot","PetalBot","SemrushBot","QuantBot","Qwantbot"]}
{"domain":"kanald.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"16personalities.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vidyard.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["ClaudeBot","Claude-User","Google-Extended","*"]}
{"domain":"lobstertube.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"ksycdn.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"boyfriend.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AhrefsBot","PetalBot"]}
{"domain":"zephr.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"google.com.gi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"aryion.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":11,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","Yandex"]}
{"domain":"sssinstagram.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wvu.edu","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"brightree.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"hawaii.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"partsouq.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"wakelet.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"idokep.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","facebookexternalhit"]}
{"domain":"chsi.com.cn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"melonstube.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"zoologyfibre.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"pornolab.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["archive.org_bot","bingbot","DuckDuckBot","Googlebot","Googlebot-Mobile","ia_archiver","Yandex","Adsbot-Google","AhrefsBot","Googlebot-Image","Mail.Ru","Mediapartners-Google","msnbot","msnbot-media","Slurp","StackRambler","Yahoo-MMCrawler","*"]}
{"domain":"r-line.ru","fetch_ok":false,"usable":false,"reason":"other","status":411,"error":null,"attempts":[411,411]}
{"domain":"antena3.ro","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"totogaming.am","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"geny.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"falabella.com.pe","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"nova.rs","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"91porn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lexmark.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"soinluck.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"nwu.ac.za","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"imgflip.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"suse.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","EasouSpider","NaverBot-1.0"]}
{"domain":"sportpesa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"clips4sale.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"veikkaus.fi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":82,"stanza_tokens":["*","Twitterbot","facebookexternalhit","AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","Andibot","anthropic-ai","Applebot","Applebot-Extended","bedrockbot","Brightbot 1.0","Bytespider","CCBot","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Diffbot","DuckAssistBot","EchoboxBot","FacebookBot","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","MistralAI-User/1.0","MyCentralAIScraperBot","NovaAct","OAI-SearchBot","omgili","omgilibot","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot","Scrapy","SemrushBot","SemrushBot-BA","SemrushBot-CT","SemrushBot-OCOB","SemrushBot-SI","SemrushBot-SWA","Sidetrade indexer bot","TikTokSpider","Timpibot","VelenPublicWebCrawler","Webzio-Extended","wpbot","YandexAdditional","YandexAdditionalBot","YouBot"]}
{"domain":"filmix.my","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"tchibo.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mangatown.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.as","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"freewebnovel.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kavak.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"geogebra.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tickettailor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"eurogamer.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":108,"stanza_tokens":["\"008\"","AddSearchBot","AI2Bot","Ai2Bot-DeepResearchEval","Ai2Bot-Dolma","Amazonbot","AmazonBuyForMe","Amzn-SearchBot","Anchor Browser","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot","Applebot-extended","atlassian-bot","AutoRAG","AwarioSmartBot","AzureAI-SearchBot","Big Sur AI","bigsur.ai","Brandwatch","Bravebot","Brightbot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank Crawler","DDM-DCipher*","DDM-DCipher/1.0.7","DDM*","DeepSeekBot","Devin","Diffbot","Direqt Anomura","DuckAssistBot","Echobot Bot","ExaBot","FacebookBot","Factset_spyderbot","FirecrawlAgent","GPTBot","HTTrack","iAsk","iAskBot","iaskspider","ICC Crawler","ICC-Crawler","ImageSiftBot","imageSpider","Kangaroo Bot","KlaviyoAIBot","Kunato","laion-huggingface-processor","LCC","LINER Bot","LinerBot","LinkupBot","Manus-User","meta-externalagent","meta-externalfetcher","meta-webindexer","mistral.ai","MistralAI-User","netEstate Imprint Crawler","NovaAct","Novellum AI Crawl","Nutch","OAI-SearchBot","Offline Explorer","omgili","opencode","PanguBot","PeopleInc-DCipher-Scraper/1.1.0","Perplexity-User","PerplexityBot","PetalBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","Scrapy","SeekrBot","SemrushBot-OCOB","SemrushBotSwa","ShapBot","Spider","TavilyBot","TerraCotta","Timpibot","Trae","TwinAgent"]}
{"domain":"portaleargo.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"goldbet.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"eredmenyek.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"thumbtack.com","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"nettiauto.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","moget","ichiro","NaverBot","Yeti","Baiduspider","Baiduspider-video","Baiduspider-image","sogou spider","YoudaoBot","YandexBot/3.0","YandexBot","http://yandex.com/bots","Yandex","CrystalSemanticsBot","PetalBot"]}
{"domain":"sesupdate.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"iccku.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"bankbazaar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["rogerbot","dotbot","GPTBot","OpenAI","CCBot","ClaudeBot","PerplexityBot","*"]}
{"domain":"audioeye.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":[429,429,429]}
{"domain":"sxyprn.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"myhentaicomics.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mensjournal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zeronaught.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"javtiful.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bestjavporn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"assoass.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"zndns.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"wartank.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ushmm.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sport.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":275,"stanza_tokens":["*","Googlebot-News","360Spider","80legs","Aboundex","accelobot","Add\\ Catalog","aiHitBot","Alexibot","Aqua_Products","AskJeeves","asterias","awcheckBot","b2w/0.1","BackDoorBot/1.0","BacklinkCrawler","Baiduspider","Barkrowler","BecomeBot","BLEXBot","BlowFish/1.0","Bookmark search tool","BotALot","brandwatch.net","BuiltBotTough","Bullseye/1.0","BunnySlippers","Butterfly","CatchBot","Charlotte","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","Clickagy Intelligence Bot v2","CLIPish","Cliqzbot","COMODO","Comodo-Certificates-Spider","CompSpyBot","Copernic","CopyRightCheck","cosmos","crawler","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","Curious","curl","DataForSeoBot","dataprovider\\.com","DinoPing","discoverybot","DittoSpyder","DomainCrawler","dotbot","dotnetdotcom","Dow\\ Jones\\ Searchbot","dumbot","EasouSpider","EmailCollector","EmailSiphon","EmailWolf","Enterprise_Search","Enterprise_Search/1.0","EroCrawler","es","Exabot","ExtractorPro","EyeMonIT","EzineArticlesLinkScanner","Ezooms","FairAd Client","Feedpresso Content Index Bot","Flaming AttackBot","Foobot","FreeFind","FTRF\\:\\ Friendly","Gaisbot","GetRight/4.2","gigabot","GPTBot","grub","grub-client","Harvest/1.5","Hatena Antenna","hloader","http://linkfluence.com","http://mediacloud.org","http://www.SearchEngineWorld.com  bot","http://www.WebmasterWorld.com  bot","HTTP_Request","HTTP_Request2","httplib","https://developers.snap.com/robots","https://help.qwant.com/bot/","humanlinks","ia_archiver","ia_archiver/1.6","Indy\\ Library","InfoNaviRobot"]}
{"domain":"darknessporn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AdsBot-Google"]}
{"domain":"hdfilmcehennemi.nl","fetch_ok":false,"usable":false,"reason":"other","status":451,"error":null,"attempts":[451,451]}
{"domain":"fotor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","AdsBot-Google","AdsIdxBot","Google-InspectionTool","Twitterbot","facebookexternalhit","LinkedInBot"]}
{"domain":"machotube.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","AhrefsBot","MJ12bot","SemrushBot-SA","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SemrushBot-CT","SemrushBot-BM"]}
{"domain":"dichvucong.gov.vn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"oanda.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","VelenPublicWebCrawler"]}
{"domain":"yesplay.bet","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"chronicle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"cudaops.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"frhiys.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"awsdns-34.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"jwpsrv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"elvebredd.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"espn.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":11,"stanza_tokens":["claritybot","GPTBot","Google-Extended","CCBot","ChatGPT-User","anthropic-ai","Omgilibot","Omgili","FacebookBot","Bytespider","*"]}
{"domain":"ytdown.to","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"aade.gr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mamastar.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["proximic","dotbot","Barkrowler","trendictionbot","SirdataBot","ICC-Crawler","meta-externalagent","bingbot","ClaudeBot","peer39_crawler","peer39_crawler/1.0","*"]}
{"domain":"homedepot.com.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":22,"stanza_tokens":["*","bingbot","GoogleOther","GoogleOther-Image","GoogleOther-Video","APIs-Google","The Googlebot-News","Googlebot smartphone","AdsBot-Google","Googlebot","Googlebot-Image","Google-InspectionTool","Google Page Speed","Chrome-Lighthouse","Googlebot-Video","Storebot-Google","AdsBot-Google-Mobile","Mediapartners-Google","developers.google.com/*","W3C_Validator","SEMrush Crawler","Screaming Frog SEO Spider"]}
{"domain":"dxtech.ai","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tivo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fresha.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["facebookexternalhit","facebookcatalog","*"]}
{"domain":"delhivery.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iefimerida.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"list.am","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"energystar.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":3,"stanza_tokens":["usasearch","*","ClaudeBot"]}
{"domain":"famethemes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xhfuck.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"nomura.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"melbetegypt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":3,"stanza_tokens":["*","GPTBot","Google-Extended"]}
{"domain":"mtsbank.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","Googlebot","Yandex","GPTBot","PerplexityBot","Google-Extended","ClaudeBot"]}
{"domain":"confirmtkt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-26.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kisskh.do","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tiktokio.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"emlakjet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","Scrappy","Scrappy Bot","discobot","dotbot","yacybot","Adsbot-Google","AdsBot-Google-Mobile"]}
{"domain":"weebcentral.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"24chasa.bg","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pearltrees.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pozdravok.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Yandex","Googlebot"]}
{"domain":"filgoal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","Googlebot-News","Googlebot","Bingbot","Applebot","PetalBot","Bytespider","Amazonbot"]}
{"domain":"eki-net.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"cuisineaz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AmazonAdBot","*","Pinterest"]}
{"domain":"maybank2u.com.my","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bl.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cleverreach.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"a-yz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"limitlesstcg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"informer.rs","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"insta-stories-viewer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sri.gob.ec","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"oantagonista.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"awsdns-14.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"lura.live","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kingofshojo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"artfight.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"win2day.at","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"hentaicity.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"servedbyadbutler.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dnsexit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"bqsex.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["MJ12bot","GPTBot/1.0","GPTBot*","SemrushBot*","DataForSeoBot*","SeznamBot*","ImagesiftBot","AhrefsBot*","ClaudeBot*","Amazonbot*"]}
{"domain":"awsdns-41.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"creative-serving.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nexcess.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dienmayxanh.com","fetch_ok":false,"usable":false,"reason":"other","status":500,"error":null,"attempts":[500,500,500]}
{"domain":"lefrecce.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mechanize","Ant.com"]}
{"domain":"samsungpass.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ECONNRESET"]}
{"domain":"cellphones.com.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot","Googlebot-image"]}
{"domain":"bikedekho.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["grapeshot","Mediapartners-Google","*"]}
{"domain":"google.dm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"livecamrips.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"router-network.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"showroomprive.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"waterstones.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tolunastart.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"novelpia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","Googlebot","Yeti","Daum","Daumoa","Ads-Naver","facebookexternalhit"]}
{"domain":"tubidy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"tuya.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot","*"]}
{"domain":"bootstrapmade.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"kampyle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rnz.co.nz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User"]}],"stanza_token_count":40,"stanza_tokens":["*","psbot","magpie-crawler","turnitinbot","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-Web","cohere-ai","Diffbot","DuckAssistBot","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","GoogleOther","GPTBot","ICC-Crawler","img2dataset","MistralAI-User","OAI-SearchBot","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Perplexity-User","Webz.io","YouBot","facebookexternalhit","Applebot","bingbot","msnbot","Yandex","Wget"]}
{"domain":"socialmediagirls.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"linke.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"newsit.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"celebforum.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"wiki.gg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":15,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","AdsBot-Google-Mobile","AdsBot-Google","PetalBot","dotbot","AhrefsBot"]}
{"domain":"aafp.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","OmtrBot/1.0"]}
{"domain":"decathlon.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"clarivate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":101,"stanza_tokens":["*","Googlebot","Googlebot-Image","Googlebot-News","Googlebot-Video","Mediapartners-Google","AdsBot-Google","Bingbot","MSNBot","BingPreview","AdIdxBot","Slurp","Baiduspider","Baiduspider-image","Baiduspider-video","Baiduspider-news","Yandex","YandexBot","YandexImages","YandexVideo","YandexNews","YandexBlogs","YandexMedia","DuckDuckBot","Sogou","Sogou web spider","Sogou Pic Spider","Sogou head spider","Sogou Orion spider","Sogou-Test-Spider","SeznamBot","Yeti","Exabot","Twiceler","ia_archiver","Twitterbot","facebot","facebookexternalhit","facebookcatalog","meta-externalagent","meta-externalfetcher","GPTBot","ChatGPT-User","CCBot","ClaudeBot","anthropic-ai","cohere-ai","Writer","AhrefsBot","AhrefsSiteAudit","SemrushBot","DotBot","BLEXBot","Barkrowler","MJ12bot","SerpstatBot","MozDotBot","Screaming Frog SEO Spider","DomainStatsBot","ZoomBot","SiteCheckerBot","Cliqzbot","SeobilityBot","UptimeRobot","Google-Structured-Data-Testing-Tool","Google-Read-Aloud","Google-InspectionTool","sqlmap","sqlmap/1.0-dev","Acunetix","masscan","Nessus","CensysInspect","AlphaBot","Go-http-client","Offline Explorer","WebCopier","Scrapy","Nutch","Heritrix","BacklinkCrawler","008","DataForSeoBot","PetalBot","Seekport Crawler","Zoominfobot","ZoominfoBot","CrunchBot","omgili","Python-urllib","python-requests","libwww-perl","httpx","curl","Wget","OkHttp","Java","TurnitinBot","crawler4j","OpenLinkProfiler"]}
{"domain":"lotterypost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"louisville.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["SiteimproveBot","Funnelback","*"]}
{"domain":"la.lv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"steamrip.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"diymysite.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"hentaiforce.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","SEOkicks-Robot","ia_archiver","archive.org_bot","special_archiver","SemrushBot","SemrushBot-SA"]}
{"domain":"packetsdk.xyz","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","UND_ERR_CONNECT_TIMEOUT","TimeoutError"]}
{"domain":"arenascan.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"thewrap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":39,"stanza_tokens":["*","AwarioRssBot","AwarioSmartBot","Baiduspider","Barkrowler","BLEXBot","CazoodleBot","DataForSeoBot","DotBot","dotbot/1.0","Gigabot","Go-http-client","Linespider","magpie-crawler","MojeekBot","Node/simplecrawler","Nuclei","Perplexity-ai","Perplexity-User","PerplexityBot","PetalBot","PiplBot","ProRataInc","Riddler","Scrapy","SeznamBot","Sidetrade indexer bot","Sogou Spider","Timpibot","TurnitinBot","VelenPublicWebCrawler","Webzio-Extended","WikiDo","Yandexbot","YouBot","Zoominfobot","archive.is","archive.today","archive.ph"]}
{"domain":"rule34.xyz","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"falabella.com.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"javhd.today","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ezoicns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"www.gob.pe","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AhrefsBot"]}
{"domain":"zdn.vn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"yaoiscan.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"apkvision.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bonprix.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nic.es","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"cwa.gov.tw","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"huanqiu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"timebucks.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sextb.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"futurecdn.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"hot-sex-tube.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"packetsdk.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"packetsdk.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bik.gov.tr","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"heap-api.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dynamic-dns.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"EAI_AGAIN","attempts":["EAI_AGAIN","EAI_AGAIN"]}
{"domain":"saveclip.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"holadns.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"aniworld.to","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mgeko.cc","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"novelfire.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"learnings.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"centris.ca","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sena.edu.co","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"fap-guru.pro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iucn.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ipma.pt","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"qobuz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["ChatGPT-User"],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User"],"path_restricted":["Claude-SearchBot"]}],"stanza_token_count":23,"stanza_tokens":["ChatGPT-User","Perplexity-User","Claude-User","GPTBot","PerplixityBot","ClaudeBot","anthropic-ai","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","FacebookBot","Bytespider","DeepSeek","Diffbot","ImagesiftBot","cohere-ai","Yandex","YandexBot","Baiduspider","Baiduspider-video","Baiduspider-image","*"]}
{"domain":"world.co.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"netfirms.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"1porn.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"obqj2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"xsrv.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"cryoutcreations.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nykaafashion.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"your-storagebox.de","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"mixnews.lv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iskur.gov.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","Googlebot","Bingbot","YandexBot","DuckDuckBot","SemrushBot","AhrefsBot","MJ12bot","DotBot","BLEXBot","*bot*"]}
{"domain":"kurier.at","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"www.gov.gr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403]}
{"domain":"tohotheater.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"anybunny.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"91porna.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-User","anthropic-ai"],"open":[],"path_restricted":["Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":11,"stanza_tokens":["*","GPTBot","ClaudeBot","Claude-User","Google-Extended","CCBot","Bytespider","Amazonbot","anthropic-ai","PerplexityBot","YouBot"]}
{"domain":"loveread.ec","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"railyatri.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"inaproc.id","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"javlibrary.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"iti.br","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"romsfun.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"amateri.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"google.sc","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"streamlabs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ml.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"d21.team","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"matrix.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cameraprive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"skylark.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"firstpost.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"asda.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"newsbeast.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["*","grapeshot","YandexBot","SentiBot","AhrefsBot","MJ12bot","Exabot","BLEXBot","DotBot","SemrushBot","bingbot"]}
{"domain":"canalblog.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"demonicscans.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"pixad.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sfdc.cl","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-32.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wbur.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":91,"stanza_tokens":["*","AhrefsBot","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot-Extended","AwarioBot","AwarioRssBot","AwarioSmartBot","Bravebot","Bytespider","CCBot","ClaudeBot","Crawl4AI","cohere-ai","cohere-training-data-crawler","Cotoyogi","DataForSeoBot","Dataprovider Spider","DeepSeekBot","diffbot","DotBot","Factset_spyderbot","Facebookbot","FirecrawlAgent","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","Google-NotebookLM","GoogleOther","GoogleOther-Extended","GPTBot","iaskspider","ICC-Crawler","aiHitBot","img2dataset","ImagesiftBot","kagi-fetcher","Kangaroo Bot","magpie-crawler","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","meta-webindexer","MistralAI-User","MJ12bot","NovaAct","Omgili","Omgilibot","Operator","PanguBot","PhindBot","peer39_crawler","PiplBot","Qwen-LLMAgent","scoop.it","Scrapy","Screaming Frog SEO Spider","Seekr","Sentibot","SemrushBot","SemrushBot-OCOB","Sidetrade indexer bot","Timpibot","TikTokSpider","TurnitinBot","VelenPublicWebCrawler","webzio-extended","WRTNBot","ZoominfoBot","Baiduspider","YandexBot","YandexImages","PetalBot","360Spider","Mail.RU_Bot","backdorbot","BlackWidow","FlashGet","GetRight","python","urllib","Go-http-client","ChatGPT-User","PerplexityBot","DuckAssistBot","YouBot","OAI-SearchBot","Claude-User","Claude-SearchBot"]}
{"domain":"google.com.ly","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"javxxx.me","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bigthink.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"allen.in","fetch_ok":false,"usable":false,"reason":"other","status":202,"error":null,"attempts":[202]}
{"domain":"nefisyemektarifleri.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":["ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot","anthropic-ai"],"open":["Claude-User"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":132,"stanza_tokens":["Mediapartners-Google","AmazonBuyForMe","Amzn-User","ChatGPT Agent","ChatGPT-User","Claude-User","Claude-Web","Devin","Gemini-Deep-Research","Google-NotebookLM","GoogleAgent-Mariner","Manus-User","MistralAI-User","MistralAI-User/1.0","NotebookLM","NovaAct","Operator","Perplexity-User","TwinAgent","AddSearchBot","atlassian-bot","Echobox Bot","EchoboxBot","facebookexternalhit","meta-externalfetcher","Meta-ExternalFetcher","WARDBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","anthropic-ai","Applebot-Extended","bedrockbot","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","DeepSeekBot","FacebookBot","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","GPTBot","img2dataset","laion-huggingface-processor","LAIONDownloader","meta-externalagent","Meta-ExternalAgent","meta-webindexer","PanguBot","Timpibot","VelenPublicWebCrawler","Andibot","AzureAI-SearchBot","Bravebot","Claude-SearchBot","DuckAssistBot","iAskBot","iaskspider","iaskspider/2.0","IbouBot","kagi-fetcher","LinkupBot","LinerBot","OAI-SearchBot","PerplexityBot","PetalBot","PhindBot","TavilyBot","WRTNBot","YouBot","ZanistaBot","aiHitBot","amazon-kendra","Amazonbot","Amzn-SearchBot","Anomura","Applebot","Awario","bigsur.ai","Brightbot 1.0","BuddyBot","Channel3Bot","Cloudflare-AutoRAG","Crawl4AI","Crawlspace","Datenbank Crawler","Diffbot","Factset_spyderbot","FirecrawlAgent","ICC-Crawler","ImagesiftBot","imageSpider","ISSCyberRiskCrawler","Kangaroo Bot","KlaviyoAIBot"]}
{"domain":"archive-digger.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"awsdns-33.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bio.site","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"fujoho.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"datezone.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["ia_archiver","bingbot","*"]}
{"domain":"singaporeair.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"shinigami.asia","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"spiceworks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":43,"stanza_tokens":["Googlebot","googlebot-image","googlebot-mobile","MSNBot","Slurp","yahoo-mmcrawler","Twitterbot/1.0","TwitterBot 1.0","AhrefsBot","AhrefsSiteAudit","Screaming Frog","Screaming Frog SEO Spider","yahoo-blogs/v3.9","Bingbot","*","\"008\"","Amazonbot","anthropic-ai","Applebot","Applebot-extended","Bytespider","CCBot","ClaudeBot","Claude-Web","cohere-ai","Diffbot","FacebookBot","GPTBot","HTTrack","Nutch","Offline Explorer","omgili","Scrapy","YouBot","meta-externalagent","Timpibot","DuckAssistBot","Perplexity-User","Ai2Bot-Dolma","meta-externalfetcher","ChatGPT-User","PerplexityBot","OAI-SearchBot"]}
{"domain":"g4media.ro","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"onlinesbi.sbi","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","ETIMEDOUT"]}
{"domain":"ticketnew.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"technorati.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":44,"stanza_tokens":["*","ADmantX","AlphaBot","anthropic-ai","AwarioRssBot","AwarioSmartBot","BLEXBot","Buzzbot","Bytespider","CCBot","ChatGPT-User","claritybot","Claude-Web","ClaudeBot","cohere-ai","Diffbot","FacebookBot","FriendlyCrawler","Google-Extended","GPTBot","huggingface","ImagesiftBot","img2dataset","magpie-crawler","Meltwater","Neevabot","news-please","NewsNow","Nutch","omgili","omgilibot","panscient.com","Perplexity-ai","PerplexityBot","PetalBot","PiplBot","scoop.it","Scrapy","Seekr","SentiBot","SeznamBot","TurnitinBot","YouBot","ZumBot"]}
{"domain":"mangakakalot.gg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","DMCAAgent","CopyrightCrawler","AntiPiracyBot","008","Slurp","goo-crawler","RakutenBot","Teoma","Baiduspider","Sogou Spider","360Spider","YisouSpider","ia_archiver","MusoBot","Link-BusterBot"]}
{"domain":"sefon.pro","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"isipp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","SeekportBot"]}
{"domain":"profesia.sk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Mediapartners-Google"]}
{"domain":"sheinindia.in","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"goias.gov.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nikkei225jp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"erothots1.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":429,"error":null,"attempts":[429,429,429]}
{"domain":"homecenter.com.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"doviz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nodr.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"bazos.sk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","Mediapartners-Google","trovitBot","pricebot","magpie-crawler","SemrushBot","trendictionbot","BLEXBot"]}
{"domain":"xnxx-ru.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"saasexch.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","facebookexternalhit","Twitterbot","Yandex"]}
{"domain":"yaklass.ru","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"espn.com.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":11,"stanza_tokens":["claritybot","GPTBot","Google-Extended","CCBot","ChatGPT-User","anthropic-ai","Omgilibot","Omgili","FacebookBot","Bytespider","*"]}
{"domain":"hentairead.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"cic.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Mozilla/5.0 (compatible; ExaleadCloudview/6;)","ExabotEI","SiteAuditBot"]}
{"domain":"onlinejobs.ph","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"spotify.dev","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nmas.com.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AdsTxtCrawler/1.0; Teads.com"]}
{"domain":"babesource.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"lottomatica.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"chunkbase.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"moe.gov.my","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"apollopharmacy.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot-image","*"]}
{"domain":"skyscanner.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["Mediapartners-Google","YisouSpider","Leikibot","ShopWiki","NPBot","psbot","HenryTheMiragoRobot","gigabot","appie","Meta-ExternalAgent","AhrefsBot","*"]}
{"domain":"lit.link","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"imovelweb.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"macworld.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":37,"stanza_tokens":["*","Ai2Bot-Dolma","AI2Bot","anthropic-ai","Applebot-Extended","Arquivo-web-crawler","AwarioRssBot","AwarioSmartBot","Barkrowler","Bytespider","CCBot","ClaudeBot","cohere-ai","cohere-training-data-crawler","crawler4j","dotbot","Diffbot","FacebookBot","FriendlyCrawler","GPTBot","https://hada.news","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","MJ12bot","PanguBot","Scrapy","Seekr","Timpibot","VelenPublicWebCrawler","Webzio-Extended"]}
{"domain":"syndigo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"diariodocomercio.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":4,"stanza_tokens":["*","CCBot","Google-Extended","GPTBot"]}
{"domain":"yenicaggazetesi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fhofficial.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"humblebundle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"skyscanner.co.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["Mediapartners-Google","YisouSpider","Leikibot","ShopWiki","NPBot","psbot","HenryTheMiragoRobot","gigabot","appie","Meta-ExternalAgent","AhrefsBot","*"]}
{"domain":"privateproperty.co.za","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Mediapartners-Google"]}
{"domain":"sss.gov.ph","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lahuashanbx.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","UND_ERR_CONNECT_TIMEOUT"]}
{"domain":"shopifyapps.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ketchjs.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"adobeccstatic.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ouo.press","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sodimac.cl","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"zonaprop.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"smbcnikko.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ucf.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","facebookexternalhit"]}
{"domain":"intsig.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","TimeoutError"]}
{"domain":"xxx-sex.one","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tokyo-sports.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":13,"stanza_tokens":["*","CCBot","GPTBot","ChatGPT-User","Google-Extended","anthropic-ai","cohere-ai","PerplexityBot","Applebot-Extended","Grok","OAI-SearchBot","Perplexity-ai","Perplexity-User"]}
{"domain":"akadns88.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"att-idns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"shonenjumpplus.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"diabetes.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"danword.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sport24.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ninthdecimal.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR"]}
{"domain":"emojicombos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"cengage.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","meta-externalagent","GPTBot"]}
{"domain":"sex6x.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"klerk.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"rmf.fm","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nitrotype.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mijnwoordenboek.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":560,"stanza_tokens":["*","Googlebot-Image","360Spider","404checker","404enemy","80legs","Abonti","Aboundex","Aboundexbot","Acunetix","ADmantX","adscanner","AfD-Verbotsverfahren","AhrefsBot","AIBOT","AiHitBot","Aipbot","Alexibot","Alligator","AllSubmitter","AlphaBot","Anarchie","Apexoo","archive.org_bot","ASPSeek","Asterias","Attach","autoemailspider","BackDoorBot","Backlink-Ceck","backlink-check","BacklinkCrawler","BackStreet","BackWeb","Badass","Bandit","Barkrowler","BatchFTP","Battleztar Bazinga","BBBike","BDCbot","BDFetch","BetaBot","Bigfoot","Bitacle","Blackboard","Black Hole","BlackWidow","BLEXBot","Blow","BlowFish","Boardreader","Bolt","BotALot","Brandprotect","Brandwatch","Buddy","BuiltBotTough","BuiltWith","Bullseye","BunnySlippers","BuzzSumo","Calculon","CATExplorador","CazoodleBot","CCBot","Cegbfeieh","CheeseBot","CherryPicker","CheTeam","ChinaClaw","Chlooe","Claritybot","Cliqzbot","Cloud mapping","coccocbot-web","Cogentbot","cognitiveseo","Collector","com.plumanalytics","Copier","CopyRightCheck","Copyscape","Cosmos","Craftbot","crawler4j","crawler.feedback","crawl.sogou.com","CrazyWebCrawler","Crescent","CrunchBot","CSHttp","Curious","Custo","DatabaseDriverMysqli","DataCha0s","DBLBot","demandbase-bot","Demon","Deusu"]}
{"domain":"sport-fm.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lgappstv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"flashscore.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"ruhr24.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":248,"stanza_tokens":["*","Amazonbot","AlibabaBot","Ai2Bot-Dolma","Applebot-Extended","Bytespider","CCBot","ChatGLM-Spider","ClaudeBot","Claude-Web","CloudVertexBot","cohere-training-data-crawler","Cotoyogi","Datenbank Crawler","Diffbot","FacebookBot","Google-Extended","GoogleOther","GPTBot","ICC-Crawler","imageSpider","Kangaroo Bot","laion-huggingface-processor","LCC","meta-externalagent","netEstate Imprint Crawler","omgili","PanguBot","PerplexityBot","SBIntuitionsBot","Spider","Timpibot","VelenPublicWebCrawler","webzio-extended","YouBot","008","Dataprovider.com","dcrawl","HelloworkJobPostingBot","HTTrack","HTTrack 3.0","KrawlerBot","MetaInspector","MetaJobBot","newspaper","Nutch","Offline Explorer","OpenindexSpider","Potions","Scrapy","ServerHunterSpider","StatsDroneBot","CloudflareBrowserRenderingCrawler","aa","AhrefsBot","Attracta","Barkrowler","BrightEdge Crawler","BLEXBot","CaliberBot","Caliperbot","ClarityBot","Clearscopebot","Cloudtrellis","cludo.com","Cocolyzebot","cognitiveSEO Crawler","contentking","Convermax","Cxense","DataForSeoBot","DataForSEO Bot","DomainStatsBot","DotBot","Dragonbot","Huckabot","Huckabuy Bot","hypestat","iaskspider","img2dataset","James BOT","LinkCheck by Siteimprove","LinkChecker Bot","linkchecker.pro","linkdexbot","LinksIndexerBot","MarketGoo","MBCrawler","MegaIndex.ru","MJ12bot","Moz dotbot","Moz rogerbot","Nitro-","NitroBot","online-webceo-bot","Prerender","Pro Sitemaps","Readable","RevvimGort","rogerbot-crawler"]}
{"domain":"bocongan.gov.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.com.kw","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"google.md","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"argenprop.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"namequery.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nhentai.xxx","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"digisport.ro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dotnxdomain.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"nottingham.ac.uk","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"sucursalelectronica.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"pluxee.app","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"adultfuns.club","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"xn--mgbkt9eckr.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"admatic.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gimkit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"adafruit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["PowerMapper","*"]}
{"domain":"solidbroker.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"snooguts.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"samarth.edu.in","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"courtlistener.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"printables.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fringster.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","AhrefsBot","SemrushBot","MJ12bot","DotBot","BLEXBot","PetalBot","Bytespider","CCBot"]}
{"domain":"autozone.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":84,"stanza_tokens":["BacklinkCrawler","GSiteCrawler","ip-web-crawler.com","ips-agent","Linkdex","linkdex.com","linkdex.com/v2.0","linkdex.com/v2.1","MJ12bot","SpyFu","Xenu","Xenu Link Sleuth","dotbot","bingbot","adidxbot","MicrosoftPreview","Slurp","COMODOSpider/Nutch-1.2","MJ12bot/v1.4.3","plukkie","Acoon","careerbot","findlinks/2.6","OpenWebIndex/Nutch-1.6","OpenWebIndex","SistrixCrawler","Sistrix","SISTRIX Crawler","SISTRIX","psbot","KaloogaBot","SeznamBot","SeznamBot/1.0","SeznamBot/1.1","SeznamBot/3.0","Baiduspider","Baiduspider-video","Baiduspider-image","Baiduspider+","Baiduspider+(+http://www.baidu.com/search/spider.htm)","Baiduspider/2.0;+http://www.baidu.com/search/spider.html","Baiduspider/2.0","+Baiduspider","+Baiduspider/2.0","+Baiduspider/2.0;++http://www.baidu.com/search/spider.html","Mozilla/5.0(compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)","EasouSpider","JikeSpider","Sosospider+","Sosospider","Sosospider/2.0","360Spider","YisouSpider","YoudaoBot/1.0","sogou spider","Exabot/3.0","Exalead","ExaLead Crawler","Wada.vn","Wada.vn Vietnamese Search","Wada.vn Vietnamese Search/2.1","NaverBot","Yeti","Yandex","Yandex/1.01.001","YandexBot/3.0-MirrorDetector","YandexImages/3.0","YandexSomething/1.","Yandex.com","YandexBot/3.0","aiHitBot","aiHitBot/1.0","aiHitBot/1.1","Mail.RU_Bot/2.0","Mail.RU","Mail.Ru","Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots","moget","ichiro","Ichiro","Ichiro 3.0","Arachmo","Zao","*"]}
{"domain":"beta-bank.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"vpsvc.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"ndtv.in","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"bcv.org.ve","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_VERIFY_LEAF_SIGNATURE","attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE","TimeoutError"]}
{"domain":"prosport.ro","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"p-cdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"awsdns-15.co.uk","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"meusitehostgator.com.br","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"javdb.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"awsdns-28.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"skbkontur.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cvte.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"olist.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iscorp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lcw.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"mangak.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"photoacompanhantes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["AhrefsBot","YandexBot","BLEXBot","Qwantify","ia_archiver","*"]}
{"domain":"anandabazar.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"hotebonytube.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"tubidy.cool","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"googll.store","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"kingmods.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"streamrail.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"talon-sec.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"vsco.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":11,"stanza_tokens":["Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Google-Extended","GPTBot","meta-externalagent","PetalBot","TikTokSpider","*"]}
{"domain":"loteriasyapuestas.es","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"fotoscapes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"clicktripz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":17,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","PerplexityBot","Amazonbot","ClaudeBot","Omgilibot","FacebookBot","Applebot","anthropic-ai","Bytespider","Claude-Web","Diffbot","ImagesiftBot","Omgili","YouBot"]}
{"domain":"javbus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gree.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Slurp","DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)"]}
{"domain":"jugabet.cl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"observeit.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-51.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"spring.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-62.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"axonixgrid.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"jiathis.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"blackmagicdesign.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":30,"stanza_tokens":["AhrefsBot","BUbiNG","dotbot","mj12bot","MJ12bot","deepcrawl","Scrapy","grub-client","k2spider","WebReaper","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","*"]}
{"domain":"oppofind.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"91porny.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Twitterbot"]}
{"domain":"7mmtv.sx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","archive.org_bot","ia_archiver","ArchiveBot","CCBot"]}
{"domain":"linksys.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mattermost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"onward.co.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"4kwallpapers.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Mediapartners-Google"]}
{"domain":"vbet.am","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"france.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","anthropic-ai"],"open":[],"path_restricted":["Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":21,"stanza_tokens":["MistralAI-User","Claude-User","Applebot","GPTBot","ChatGPT-User","CCBot","Google-Extended","anthropic-ai","Claude-Web","ClaudeBot","Bytespider","Amazonbot","OAI-SearchBot","PerplexityBot","Applebot-Extended","cohere-ai","cohere-training-data-crawler","FacebookBot","Meta-ExternalAgent","AhrefsBot","*"]}
{"domain":"infomart.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.com.et","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"sail-personalize.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"typingtest.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":12,"stanza_tokens":["Bytespider","GPTBot","claudebot","panscient","PerplexityBot","ImagesiftBot","Amazonbot","Applebot-Extended","CCBot","Diffbot","FacebookBot","omgili"]}
{"domain":"cool18.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"readli.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Bingbot"]}
{"domain":"playmods.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["Baiduspider","Sosospider","sogou spider","YodaoBot","360spider","Bytespider","Bingbot","*"]}
{"domain":"highwebmedia.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bricks-co.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"pathofexile.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","bingbot"]}
{"domain":"vseosvita.ua","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bodegaaurrera.com.mx","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"thethaovanhoa.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"happy-porn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":3,"stanza_tokens":["GPTBot","ClaudeBot","*"]}
{"domain":"delishkitchen.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"1905.com","fetch_ok":false,"usable":false,"reason":"other","status":409,"error":null,"attempts":[409,409]}
{"domain":"qprod2.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"drikpanchang.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["*","cohere-ai","YouBot","SeekportBot","ZoominfoBot","Barkrowler","MegaIndex","SeznamBot","Diffbot","CCBot","Linespider","dotbot","PetalBot","Bytespider","Sogou web spider","Sogou inst spider","Baiduspider","CriteoBot"]}
{"domain":"gigabyte.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"search-redirect.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"mangalib.me","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"betmexico.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-36.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"snplow.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"anker.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Googlebot","Googlebot-image","*"]}
{"domain":"ok.porn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"statsig.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yellowpages.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"motorsport.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["Mediapartners-Google","Yandex","Googlebot","proximic","AmazonAdBot","Amazon-Advertising-ad-standards-bot/1.0","APS-ad-standards-bot/1.0","*"]}
{"domain":"ntvspor.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":12,"stanza_tokens":["*","GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","PerplexityBot","TurnitinBot","cohere-ai","OAI-SearchBot","http://quillbot.com","Google-Extended"]}
{"domain":"1377x.to","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"lokmat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["facebookexternalhit","*"]}
{"domain":"google.com.sv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"apuestasroyal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"mercantilbanco.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ifit.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"annualreviews.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["GoogleBot","YandexBot","YandexImages","bingbot","Baiduspider","*"]}
{"domain":"urban-vpn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"delicious.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zooplus.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wpeproxy.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wireguard.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"realtimetrains.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["Googlebot","Googlebot-Image","Bingbot","AdIdxBot","BingPreview","DuckDuckBot","Baiduspider","Yandex","Applebot","msnbot","Naverbot","Slurp","teoma","facebookexternalhit","Twitterbot","LinkedInBot","Pinterest","WhatsApp","Pubmatic Crawler Bot","*"]}
{"domain":"gladly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","Googlebot","Bingbot","Applebot","DuckDuckBot","GPTBot","ChatGPT-User","ClaudeBot","Claude-User","Google-Extended","CCBot","Meta-ExternalAgent","Applebot-Extended","PerplexityBot","YouBot","Twitterbot"]}
{"domain":"gammaplatform.com","fetch_ok":false,"usable":false,"reason":"other","status":204,"error":null,"attempts":[204]}
{"domain":"splashtop.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","SemrushBot","SemrushBot-SA"]}
{"domain":"p-world.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","KDDI-Googlebot-Mobile","bingbot","dotbot","Bytespider"]}
{"domain":"yts.bz","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"roman.co.uk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"line.biz","fetch_ok":false,"usable":false,"reason":"other","status":204,"error":null,"attempts":[204]}
{"domain":"transip.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cinecalidad.am","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"newsbomb.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AhrefsBot"]}
{"domain":"wav.tv","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"wri.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"natomanga.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","DMCAAgent","CopyrightCrawler","AntiPiracyBot","008","Slurp","goo-crawler","RakutenBot","Teoma","Baiduspider","Sogou Spider","360Spider","YisouSpider","ia_archiver","MusoBot","Link-BusterBot"]}
{"domain":"cairn.info","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"uvm.edu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gree.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"namirial.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"t-bank-app.su","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","ENOTFOUND"]}
{"domain":"mercdn.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"fanmtl.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"lecturas.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"freeporno.xxx","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","AhrefsBot","MJ12bot","SemrushBot-SA","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SemrushBot-CT","SemrushBot-BM"]}
{"domain":"cnn.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AhrefsBot","SemrushBot"]}
{"domain":"questionablequesting.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"amazonses.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"google.com.mt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"alo.bg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":17,"stanza_tokens":["MJ12bot","SemrushBot","SemrushBot-SA","serpstatbot","dotbot","BLEXBot","barkrowler","RSiteAuditor","CLASSLA-web","ImagesiftBot","PetalBot","Paqlebot","Baiduspider","*","Bingbot","MSNBot","BingPreview"]}
{"domain":"jansatta.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"files.fm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"deti-online.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"google.co.bw","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"betway.co.mz","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"filmmakinesi.to","fetch_ok":false,"usable":false,"reason":"other","status":451,"error":null,"attempts":[451,451]}
{"domain":"supergamesky.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bluesystem.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"shaalaa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":168,"stanza_tokens":["*","Baiduspider","Baiduspider+","Baiduspider+(+http://www.baidu.com/search/spider.htm)","Baiduspider/2.0;+http://www.baidu.com/search/spider.html","Baiduspider/2.0","+Baiduspider","+Baiduspider/2.0","+Baiduspider/2.0;++http://www.baidu.com/search/spider.html","Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)","spider","Yandex","Yandex.com","YandexBot/3.0","Sosospider+","ezooms","Ezooms/1.0","Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot[at]gmail[dot]com)","sogou spider","Wotbox/2.01","dwbot","Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)","OmniExplorer_Bot","FreeFind","BecomeBot","Nutch","Jetbot/1.0","Jetbot","WebVac","Stanford","naver","dumbot","Hatena Antenna","grub-client","grub","WebZip","larbin","b2w/0.1","Copernic","psbot","Python-urllib","NetMechanic","URL_Spider_Pro","CherryPicker","EmailCollector","EmailSiphon","WebBandit","EmailWolf","ExtractorPro","CopyRightCheck","Crescent","SiteSnagger","ProWebWalker","CheeseBot","LNSpiderguy","Alexibot","Teleport","TeleportPro","Stanford Comp Sci","MIIxpc","Telesoft","Website Quester","moget/2.1","WebZip/4.0","WebStripper","WebSauger","WebCopier","NetAnts","Mister PiX","WebAuto","TheNomad","WWW-Collector-E","RMA","libWeb/clsHTTP","asterias","httplib","turingos","spanner","InfoNaviRobot","Harvest/1.5","Bullseye/1.0","Crescent Internet ToolPak HTTP OLE Control v.1.0","CherryPickerSE/1.0","CherryPickerElite/1.0","WebBandit/3.50","NICErsPRO","Microsoft URL Control - 5.01.4511","DittoSpyder","Foobot","SpankBot","BotALot","lwp-trivial/1.34","lwp-trivial","BunnySlippers","Microsoft URL Control - 6.00.8169","URLy Warning","Wget/1.6","Wget/1.5.3","Wget","LinkWalker"]}
{"domain":"rawstory.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":6,"stanza_tokens":["*","CCBot","GPTBot","Google-Extended","PerplexityBot","ias_crawler"]}
{"domain":"bom.gov.au","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"justanswer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":19,"stanza_tokens":["Xombot","AhrefsBot","Zing-BottaBot","Zing-BottaBot/2.0","GoScraper","GPTBot","Google-Extended","dotbot","MJ12bot","BLEXBot","trendictionbot","Barkrowler","Turnitin","MojeekBot","BingBot","PerplexityBot","ChatGPTBot","GeminiBot","*"]}
{"domain":"ddnavi.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND",403]}
{"domain":"diginetica.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"just.ro","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"uni-freiburg.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kissjav.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"epey.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":16,"stanza_tokens":["Claude-User","Claude-Web","Claude-SearchBot","ChatGPT-User","OAI-SearchBot","PerplexityBot","Perplexity-User","GPTBot","ClaudeBot","CCBot","Google-Extended","Bytespider","meta-externalagent","bingbot","ia_archiver","*"]}
{"domain":"alaska.gov","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","PiplBot"]}
{"domain":"dns.nl","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"traveltowngame.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"stheadline.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Googlebot","Googlebot-News","Bingbot","Slurp","Twitterbot","facebookexternalhit","Applebot","baiduspider"]}
{"domain":"flashscore.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"yiv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"orange.mg","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",404]}
{"domain":"royalcaribbean.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"qxbroker.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"eyemedvisioncare.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"anyrtb.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"rosdomofon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","Googlebot"]}
{"domain":"huyatvns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"momvids.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":16,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","Mail.Ru","Baiduspider","MSNBot","Bingbot","Slurp","Yandex"]}
{"domain":"dola.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gismeteo.by","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"sfile.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fastpic.ru","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"okiloveyoupleasedonttouchme.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ENOTFOUND"]}
{"domain":"vhlcentral.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"walesonline.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":[],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":16,"stanza_tokens":["*","semetrical","GPTBot","OAI-SearchBot","Applebot-Extended","PerplexityBot","Perplexity-ai","CCBot","YouBot","ClaudeBot","anthropic-ai","Claude-Web","grapeshot","AmazonAdBot","ozone","Meta-ExternalAgent"]}
{"domain":"f-secure.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"transmit.live","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ibtimes.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"privateinternetaccess.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Googlebot","Googlebot-image","AdsBot-Google","Twitterbot"]}
{"domain":"futura-sciences.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":["anthropic-ai"]}],"stanza_token_count":19,"stanza_tokens":["GPTBot","ClaudeBot","Amazonbot","Applebot","Applebot-Extended","CCbot","Bytespider","PetalBot","FacebookBot","MegaIndex.ru","OAI-SearchBot","ChatGPT-User","Claude-User","Claude-SearchBot","Amzn-Searchbot","Amzn-User","AmazonAdBot","PerplexityBot","*"]}
{"domain":"storysaver.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hello.porn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"romeo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"parfois.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"apuestatotal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"exodus.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"snapp.ir","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TypeError","attempts":["TypeError","TimeoutError"]}
{"domain":"rudderlabs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"orizzontescuola.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"wolvden.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"indiandefencereview.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":23,"stanza_tokens":["*","8LEGS","AhrefsBot","AspiegelBot","BLEXBot","Barkrowler","DotBot","MJ12bot","MauiBot","Nimbostratus-Bot","PetalBot","SemrushBot","SeznamBot","Sogou","serpstatbot","trendiction","TextBulkerBot","Amazonbot","Bytespider","CCBot","cohere-ai","FacebookBot","Omgilibot"]}
{"domain":"amazon.jobs","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["AhrefsBot","*"]}
{"domain":"google.bj","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"google.com.sb","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"panda.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"dedeman.ro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"8888.bg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"juris.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"paykeeper.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pipedrive.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"runsignup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["YandexBot","dotbot","SemrushBot","SemrushBot-SA","VegeBot","Vegi bot","BaiduSpider","BLEXBot","panscient.com","AhrefsBot","PiplBot","AlphaSeoBot","PetalBot","facebookexternalhit","*"]}
{"domain":"nikkeibp.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":36,"stanza_tokens":["*","Claude-User","FacebookBot","meta-externalfetcher","CCBot","cohere-ai","Claude-Web","PiplBot","GPTBot","MuckRack","PerplexityBot","magpie-crawler","Google-Extended","OAI-SearchBot","ClaudeBot","omgilibot","Gensparkbot","ICC-Crawler","MJ12bot","Timpibot","Webzio","Perplexity-User","cohere-training-data-crawler","Scrapy","Diffbot","omgili","Applebot-Extended","Meta-ExternalAgent","Claude-SearchBot","Bytespider","AmazonBot","ChatGPT-User","Perplexity-ai","Webzio-Extended","Webzio-extended","anthropic-ai"]}
{"domain":"sefan.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":39,"stanza_tokens":["*","AhrefsBot","Aliyun","AliyunSecBot","AliyunSecBot/Aliyun","Amazonbot","Amzn-SearchBot","Baiduspider","barkrowler","BLEXBot","Brightbot 1.0","Bytespider","CCBot","ClaudeBot","Claude-Web","Cliqzbot","coccocbot","DataForSeoBot","dotbot","GPTBot","ImagesiftBot","Linguee","megaindex","meta-externalagent","Nimbostratus","oBot","PerplexityBot","PetalBot","proximic","Seekport Crawler","SemrushBot","serpstatbot","SleepBot","SMTBot","Sogou web spider","SurdotlyBot","TikTokSpider","Turnitin Robot","YouBot"]}
{"domain":"pudelek.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]}],"stanza_token_count":265,"stanza_tokens":["*","008","010","360Spider","80legs","Aboundex","accelobot","Add\\ Catalog","AhrefsBot","aiHitBot","Alexibot","Aqua_Products","AskJeeves","asterias","awcheckBot","b2w/0.1","BackDoorBot/1.0","BacklinkCrawler","Baiduspider","BecomeBot","BLEXBot","BlowFish/1.0","Bookmark search tool","BotALot","brandwatch.net","BuiltBotTough","Bullseye/1.0","BunnySlippers","Butterfly","CatchBot","Charlotte","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","CLIPish","Cliqzbot","COMODO","Comodo-Certificates-Spider","CompSpyBot","Copernic","CopyRightCheck","cosmos","crawler","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","Curious","curl","dataprovider\\.com","DinoPing","discoverybot","DittoSpyder","DomainCrawler","dotbot","dotnetdotcom","Dow\\ Jones\\ Searchbot","dumbot","EasouSpider","EmailCollector","EmailSiphon","EmailWolf","Enterprise_Search","Enterprise_Search/1.0","EroCrawler","es","Exabot","ExtractorPro","EzineArticlesLinkScanner","Ezooms","FairAd Client","Flaming AttackBot","Foobot","FreeFind","FTRF\\:\\ Friendly","Gaisbot","GetRight/4.2","gigabot","grub","grub-client","Harvest/1.5","Hatena Antenna","hloader","http://www.SearchEngineWorld.com bot","http://www.WebmasterWorld.com bot","HTTP_Request","HTTP_Request2","httplib","humanlinks","ia_archiver","ia_archiver/1.6","Indy\\ Library","InfoNaviRobot","ip\\-web\\-crawler\\.com","Iron33/1.0.2","Jakarta\\ Commons-HttpClient","Jeeves","JennyBot","Jetbot","Jetbot/1.0","JikeSpider"]}
{"domain":"wildlifestudios.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"claspws.tv","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hayamp.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"thespacemachines.st","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ENOTFOUND"]}
{"domain":"jjgirls.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"libguides.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"self.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","FacebookExternalHit","Twitterbot","Pinterestbot","LinkedInBot","TikTokSpider","Storebot-Google","Google-InspectionTool","AmazonAdBot","Google-Extended","Google-CloudVertexBot","GoogleOther","Applebot-Extended","Amazonbot","meta-webindexer","meta-externalagent","meta-externalfetcher","ClaudeBot","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","cohere-training-data-crawler","cohere-ai","CCBot","PanguBot","PetalBot","Bytespider","Diffbot","DuckAssistBot","MistralAI-User","Timpibot","Webzio","Webzio-Extended","archive.org_bot","ia_archiver","ia_archiver-web.archive.org","heritrix"]}
{"domain":"cfe.mx","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"afp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"supersportbet.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pubnubapi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dopa.go.th","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"online-reservations.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"smartnews-ads.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sport.ro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":3,"stanza_tokens":["*","SeekportBot","GPTBot"]}
{"domain":"brainyquote.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"flashscore.sk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"flashscore.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"sgsnssdk.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-02.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"unilever.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"servicenow.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"curalate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-11.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"bd33fgabh.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sphinx-doc.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"myheritage.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":152,"stanza_tokens":["*","Anthropic-ai","ChatGPT-User","ClaudeBot","Claude-Web","Cohere-ai","GPTBot","GPT","ImagesiftBot","PerplexityBot","YouBot","AndiBot","MindsDBBot","MetaphorSystemsBot","PhindBot","WolframAlphaBot","oai-searchbot","You.com","Applebot-Extended","Google-Extended","Seekr","MistralAI-User","ChatDK-Bot","SBIntuitionsBot","DuckAssistBot","quillbot","MojeekBot","Yandex","YaDirectFetcher","Baiduspider","Sosospider","YodaoBot","Yeti","Sogou","Bytespider","Mail.Ru","SeznamBot","coccocbot-web","AhrefsBot","AhrefsSiteAudit","SemrushBot","MajesticBot","MozBot","UbersuggestBot","serpstatbot","BrightEdgeBot","SiteAuditBot","cognitiveSEO","DataForSeoBot","RankActiveBot","SpyFuBot","SEOlyticsCrawler","RankurBot","RyteBot","Monsidobot","Cocolyzebot","siteimprove.com/bots","MOSBot","BLEXBot","DotBot","MJ12bot","Diffbot","Diffbot/1.0","Diffbot/2.0","ZoominfoBot","intelx.io_bot","SupabaseEdgeScraper","SaaSBrowserBot","LinkupBot","domainsbot","URLChecker","URLChecker/1.0","URLChecker/2.0","URLCheckerBot","fynd.bot","MatchMyVC-bot","affiliate-wiki-bot","ScannerBot","WebAuditScannerBot","Vladhog","WellKnownBot","d-bot-health-check","uptimerobot","Thinkbot","Newslitbot","GnowitNewsbot","Poggio-Citations","ScourRSSBot","smry.ai","rss-is-dead.lol","rss-is-dead.lol web bot","Paqlebot","SeekportBot","Cubot","yacybot","Dragonbot","Dubbotbot","EzoicBot-Nicheiq","SurdotlyBot","TSMbot"]}
{"domain":"betano.pe","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.kg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"distintas.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"gemidos.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"lepoint.fr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bhvrdbd.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"myna.go.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"bibliaon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"officeally.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"freejobalert.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":17,"stanza_tokens":["*","GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","anthropic-ai","Claude-Web","PerplexityBot","Google-Extended","CCBot","Bytespider","FacebookBot","Meta-ExternalAgent","Applebot-Extended","Amazonbot","cohere-ai","DuckAssistBot"]}
{"domain":"tumbex.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.co.ug","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"redwap.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"therealreal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"portalsamorzadowy.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":318,"stanza_tokens":["*","Bingbot","Yandex","008","010","360Spider","80legs","Aboundex","accelobot","Add\\ Catalog","AhrefsBot","aiHitBot","Alexibot","Aqua_Products","AskJeeves","asterias","awcheckBot","b2w/0.1","BackDoorBot/1.0","BacklinkCrawler","Baiduspider","Barkrowler","BecomeBot","BLEXBot","BlowFish/1.0","Bookmark search tool","BotALot","brandwatch.net","BrightEdge Crawler","BuiltBotTough","Bullseye/1.0","BunnySlippers","Butterfly","CatchBot","Charlotte","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","CLIPish","Cliqzbot","Cocolyzebot","COMODO","Comodo-Certificates-Spider","CompSpyBot","Copernic","CopyRightCheck","cosmos","crawler","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","Curious","curl","DataForSeoBot","dataprovider\\.com","DinoPing","discoverybot","DittoSpyder","DomainCrawler","DomainStatsBot","dotbot","dotnetdotcom","Dow\\ Jones\\ Searchbot","dumbot","EasouSpider","EmailCollector","EmailSiphon","EmailWolf","Enterprise_Search","Enterprise_Search/1.0","EroCrawler","es","Exabot","ExtractorPro","EzineArticlesLinkScanner","Ezooms","FairAd Client","Flaming AttackBot","Foobot","FreeFind","FTRF\\:\\ Friendly","Gaisbot","GetRight/4.2","gigabot","grub","grub-client","Harvest/1.5","Hatena Antenna","hloader","http://www.SearchEngineWorld.com bot","http://www.WebmasterWorld.com bot","HTTP_Request","HTTP_Request2","httplib","humanlinks","hypestat","ia_archiver","ia_archiver/1.6","Indy\\ Library","InfoNaviRobot"]}
{"domain":"trevenque.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"afrointroductions.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"irondns.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"okbet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mywellness.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Baiduspider","Baiduspider-video","Baiduspider-image","*"]}
{"domain":"bluedart.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"targetads.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"colaboraread.com.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pscube.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"monetate.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"dignitymemorial.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"presseportal.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["BurdaForward","magpie-crawler","*"]}
{"domain":"canalrcn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hentai20.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"law.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":69,"stanza_tokens":["*","AhrefsBot","AI2Bot","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","Arquivo-web-crawler","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","coccocbot-web","cohere-ai","cohere-training-data-crawler","Crawlspace","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Google-CloudVertexBot","Google-Extended","GPTBot","https://hada.news","ICC-Crawler","ImagesiftBot","https://www.imediaethics.org","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","magpie-crawler","Meta-ExternalAgent","Meta-ExternalFetcher","Mojeek","MojeekBot","MoodleBot","Netvibes","NewsNow","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","PetalBot","Scrapy","Seekr","SeznamBot","Timpibot","Turnitin","TurnitinBot","VelenPublicWebCrawler","Webzio-Extended","yacy","yacybot","Yandex","YandexAdditional","YandexAdditionalBot","YouBot","Perplexity-User","Claude-User","Claude-SearchBot","Webzio","Gemini-Deep-Research","MistralAI-User","SemrushBot-OCOB"]}
{"domain":"doramy.club","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"astronautlividlyreformer.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"ctolabperfstats.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"allegro.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"trezor.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"snapinsta.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"techopedia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Yeti","NaverBot","Mozilla/5.0 (compatible; Qwantify/; +https://www.qwant.com/)"]}
{"domain":"gzone.ph","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.tn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"ero-video.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"coupert.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"flashscore.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"google.co.vi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"toroporno.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"skyscanner.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["Mediapartners-Google","YisouSpider","Leikibot","ShopWiki","NPBot","psbot","HenryTheMiragoRobot","gigabot","appie","Meta-ExternalAgent","AhrefsBot","*"]}
{"domain":"labour.gov.za","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"shopsy.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["Mediapartners-Google","Adsbot-Google","Googlebot-Image","*"]}
{"domain":"csgoskins.gg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thetvapp.to","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"japanet.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"icm.edu.pl","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"pdscrb.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nonamesec.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ECONNREFUSED",404]}
{"domain":"niji-gazo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Twitterbot","Yeti","Baiduspider","Ezooms","bingbot"]}
{"domain":"adjarabet.am","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"manhuagui.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","Claude-SearchBot"],"open":[],"path_restricted":["Claude-User","anthropic-ai"]}],"stanza_token_count":40,"stanza_tokens":["ADmantX","MauiBot","CCBot","VoluumDSP","MediaPartners-Google","Cliqzbot","BLEXBot","Applebot","AhrefsBot","MegaIndex.ru","megaindex.com","Bytespider","serpstatbot","AspiegelBot","Baiduspider","Sosospider","sogou spider","Sogou web spider","360spider","YisouSpider","YodaoBot","MJ12bot","grapeshot","MTRobot","Barkrowler","SemrushBot","VelenPublicWebCrawler","Mail.RU_Bot","DataForSeoBot","Amazonbot","YandexBot","Leikibot","GPTBot","AwarioBot","meta-externalagent","PlurkBot","ClaudeBot","Claude-SearchBot","DotBot","*"]}
{"domain":"ampaeservices.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"procore.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"samqaicongen.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"react.dev","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"geometrylitepc.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Mediapartners-Google"]}
{"domain":"bqstreamer.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"viki.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"corevibe-network.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"megatv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"5play.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Mediapartners-Google*"]}
{"domain":"d2jsp.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bet88.ph","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bluecore.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rezultati.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"wplay.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"decathlon.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"daisonet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","adsbot-google","Nutch","AhrefsBot","AhrefsSiteAudit","MJ12bot","Pinterest"]}
{"domain":"tvonenews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cat.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"service.one","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"toast.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"slashgear.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"xanimu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AdsBot-Google"]}
{"domain":"sokmil.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","bingbot","SemrushBot"]}
{"domain":"awsdns-22.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"doublelist.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hentaivox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"myket.ir","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"foreignaffairs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":32,"stanza_tokens":["OpenAI","CCBot","ChatGPT","ChatGPT-User","GPTBot","Google-Extended","anthropic-ai","cohere-ai","ia_archiver","omgili","omgilibot","MJ12bot","PiplBot","Claude-Web","ClaudeBot","FacebookBot","Amazonbot","Bytespider","magpie-crawler","PerplexityBot","AhrefsBot","Bytedance","Diffbot","TurnitinBot","Quora-Bot","Meta-ExternalAgent","Applebot-Extended","Google-CloudVertexBot","DeepSeekBot","xAI","Grok","*"]}
{"domain":"hubcloud.foo","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"otakudesu.blog","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"prehrajto.cz","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"hqporn.xxx","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","AhrefsBot","MJ12bot","SemrushBot-SA","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SemrushBot-CT","SemrushBot-BM"]}
{"domain":"hihi2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"seg-social.pt","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nic.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"novel543.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Googlebot","Mediapartners-Google","Adsbot-Google","Bingbot","msnbot","Slurp","Twitterbot","popIn_Agent"]}
{"domain":"lesta.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"baike.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":29,"stanza_tokens":["UbiCrawler","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","wget","grub-client","k2spider","NPBot","WebReaper","Baiduspider","baiduspider","*"]}
{"domain":"rst.ua","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bustybus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","AhrefsBot","MJ12bot","SemrushBot-SA","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SemrushBot-CT","SemrushBot-BM"]}
{"domain":"partner.co.il","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"belta.by","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["googlebot-news","*"]}
{"domain":"rakuten-card.co.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cuerpomente.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"andhrajyothy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Mediapartners-Google","AdsBot-Google"]}
{"domain":"google.lu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"punishworld.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AdsBot-Google"]}
{"domain":"camscanner.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"openssp.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"trafficnews.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"blurb.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":36,"stanza_tokens":["DigExt","DTS Agent","Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt; DTS Agent","The Knowledge AI","heritrix","heritrix/3.3.0","CRAZYWEBCRAWLER","metadataparser","seoscanners","SEOkicks-Robot","OpenLinkProfiler","MegaIndex","id-search","istellabot","centurybot9@gmail.com","~crawler","test bot","ZumBot","Wotbox","YisouSpider","G-i-g-a-b-o-t","yacybot","s~snapchat-proxy","GnowitNewsbot","PaperLiBot","BUbiNG","Applebot","baiduspider","Bingbot","Googlebot","msnbot","Naverbot","Twitterbot","Yandex","Slurp","*"]}
{"domain":"ssstwitter.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"windscribe.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"twittervideodownloader.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["Mediapartners-Google"]}
{"domain":"startech.com.bd","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"milli.gold","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TypeError","TypeError"]}
{"domain":"gumtree.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Mediapartners-Google"]}
{"domain":"teenxy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":22,"stanza_tokens":["*","MSNBot","Bingbot","YandexBot","Baiduspider","Sogou","SemrushBot","AhrefsBot","BLEXBot","MJ12bot","008","sistrix","ia_archiver","proximic","SiteExplorer","SemrushBot-SA","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SemrushBot-CT","SemrushBot-BM","SplitSignalBot"]}
{"domain":"supersport.hr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"leroymerlin.pt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":50,"stanza_tokens":["*","expo9","Orthogaffe","UbiCrawler","DOC","Zao","Zealbot","MSIECrawler","SiteSnagger","WebStripper","Fetch","OfflineExplorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Xenu","larbin","libwww","ZyBORG","DownloadNinja","wget","grub-client","WebCopier","Python-urllib","URL_Spider_Pro","EmailCollector","EmailSiphon","WebBandit","EmailWolf","ExtractorPro","CopyRightCheck","Alexibot","Zeus","ZeusLinkScout","EroCrawler","LinkScan","WebCapture 2.0","EtaoSpider","Black Hole","Yahoo! Slurp China","Yandex","YandexTurbo","YandexBot","msnbot","Slurp","Nutch","008","CherryPicker"]}
{"domain":"keibalab.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"upsdc.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"news-img.site","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nya5.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"massagerepublic.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"glorioushop.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"convertexperiments.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hoodamath.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"truist.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fdating.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"google.ml","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"google.pn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"garotacomlocal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tpmn.io","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"trustx.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"awsdns-23.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"flashscore.co.id","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"google.com.sl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"headspace.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Twitterbot","*"]}
{"domain":"redwap.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":26,"stanza_tokens":["*","YandexVideoParser","008","BLEXBot","AhrefsBot","Vagabondo","SEOkicks-Robot","ia_archiver","archive.org_bot","special_archiver","MJ12bot","heritrix","netEstate NE Crawler","sistrix","WBSearchBot","QuerySeekerSpider","proximic","SiteExplorer","SemrushBot","SemrushBot-SA","Mail.Ru","Baiduspider","MSNBot","Bingbot","Slurp","Yandex"]}
{"domain":"google.ki","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"toytheater.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kniga-online.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"zbporn.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AhrefsBot"]}
{"domain":"emc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"reliancedigital.in","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"eyefinity.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"esakal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AdsBot-Google","Mediapartners-Google"]}
{"domain":"pomorska.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":9,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"diva-portal.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["Baiduspider","Yandex","Mail.Ru","AhrefsBot","coccoc","SemrushBot","SemrushBot-SA","Scrapy","Riddler","SemanticScholarBot","*"]}
{"domain":"bangkokpost.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Googlebot-News","*","grapeshot"]}
{"domain":"fastebridge.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"wowscale.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"stamped.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rip.ie","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":123,"stanza_tokens":["AddSearchBot","AI2Bot","AI2Bot-DeepResearchEval","Ai2Bot-Dolma","aiHitBot","amazon-kendra","Amazonbot","AmazonBuyForMe","Amzn-SearchBot","Amzn-User","Andibot","Anomura","anthropic-ai","ApifyBot","ApifyWebsiteContentCrawler","Applebot","Applebot-Extended","Aranet-SearchBot","atlassian-bot","Awario","AzureAI-SearchBot","bedrockbot","bigsur.ai","Bravebot","Brightbot 1.0","BuddyBot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT Agent","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","Cloudflare-AutoRAG","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Crawlspace","Datenbank Crawler","DeepSeekBot","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","ExaBot","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","GPTBot","iAskBot","iaskspider","iaskspider/2.0","IbouBot","ICC-Crawler","ImagesiftBot","imageSpider","img2dataset","ISSCyberRiskCrawler","kagi-fetcher","Kangaroo Bot","KlaviyoAIBot","KunatoCrawler","laion-huggingface-processor","LAIONDownloader","LCC","LinerBot","Linguee Bot","LinkupBot","Manus-User","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NotebookLM","NovaAct","OAI-SearchBot","omgili","omgilibot","OpenAI","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poggio-Citations","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot"]}
{"domain":"ncore.pro","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"tube8.fr","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"qwen.ai","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"f1.com.tw","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_UNSUPPORTED_PROTOCOL","attempts":["ERR_SSL_UNSUPPORTED_PROTOCOL","ERR_SSL_UNSUPPORTED_PROTOCOL"]}
{"domain":"ponta.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot","*"]}
{"domain":"hentaimama.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"gamerxyt.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"grapee.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"redporn.porn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","AhrefsBot","MJ12bot","SemrushBot-SA","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SemrushBot-CT","SemrushBot-BM"]}
{"domain":"betano.bet.ar","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"getmodsapk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"snapsave.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","BacklinkCrawler","Backlink-Ceck","backlink-check","MJ12bot","AhrefsBot","SemrushBot"]}
{"domain":"pciconcursos.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"loaded.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot","Googlebot-image","ia_archiver"]}
{"domain":"ojrq.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ssllabs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"romsgames.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"usmagazine.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":103,"stanza_tokens":["*","A6-Indexer","AddSearchBot","aiHitBot","Andibot","anthropic-aiBytespider","AspiegelBot","archive.org_bot","Barkrowler","bigsur.ai","BLP_bbot","Brightbot 1.0","Buck","ChatGLM-Spider","coccocbot-web","Cotoyogi","crawler4j","Crawlspace","Datenbank Crawler","cohere-training-data-crawler","cohere-ai","omgili","omgilibot","Omgilibot","Applebot-Extended","Bytespider","CCBot","ChatGPT","Diffbot","EasouSpider","Echobot Bot","EchoboxBot","ecoResearch","FirecrawlAgent","Genieo","https://hada.news","ia_archiver","ia_archiver-web.archive.org","iaskspider/2.0","ICC-Crawler","img2dataset","JenkersBot","Jetslide","Kangaroo Bot","KunatoCrawler","LivelapBot","magpie-crawler","MauiBot","MAZBot","MistralAI-User","MistralAI-User/1.0","Mojeek","MojeekBot","MyCentralAIScraperBot","news-please","NewsNow","NovaAct","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","PetalBot","PanguBot","Poseidon Research Crawler","Primalbot","QuillBot","quillbot.com","R6_CommentReader","SBIntuitionsBot","Scrapy","SeekportBot","Seekr","SeekrBot","SeznamBot","SeznamHomepageCrawler","Sidetrade indexer bot","Spinn3r","TaraGroup Intelligent Bot","Thinkbot","Timpibot","VelenPublicWebCrawler","Verity","Verity/1.1","ViennaTinyBot","WebVac","Webzio-Extended","WebZip","WRTNBot","yacy","yacybot","YouBot","ankiro.com","CheckMarkNetwork","dataprovider","legistorm.com","Magus Bot","Netvibes","nlcrawler","NLCrawler","Oracle Secure Enterprise Search","panscient.com"]}
{"domain":"puntoticket.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["Baiduspider","baiduspider","Baiduspider+","Baiduspider-video","Baiduspider-image","asterias","Spiderbot/Nutch-1.7","*"]}
{"domain":"wonporn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yanmaga.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"toyota.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gosexpod.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["msnbot","bingbot","PetalBot","BLEXBot","*"]}
{"domain":"pornohirsch.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rynekzdrowia.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":318,"stanza_tokens":["*","Bingbot","Yandex","008","010","360Spider","80legs","Aboundex","accelobot","Add\\ Catalog","AhrefsBot","aiHitBot","Alexibot","Aqua_Products","AskJeeves","asterias","awcheckBot","b2w/0.1","BackDoorBot/1.0","BacklinkCrawler","Baiduspider","Barkrowler","BecomeBot","BLEXBot","BlowFish/1.0","Bookmark search tool","BotALot","brandwatch.net","BrightEdge Crawler","BuiltBotTough","Bullseye/1.0","BunnySlippers","Butterfly","CatchBot","Charlotte","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","CLIPish","Cliqzbot","Cocolyzebot","COMODO","Comodo-Certificates-Spider","CompSpyBot","Copernic","CopyRightCheck","cosmos","crawler","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","Curious","curl","DataForSeoBot","dataprovider\\.com","DinoPing","discoverybot","DittoSpyder","DomainCrawler","DomainStatsBot","dotbot","dotnetdotcom","Dow\\ Jones\\ Searchbot","dumbot","EasouSpider","EmailCollector","EmailSiphon","EmailWolf","Enterprise_Search","Enterprise_Search/1.0","EroCrawler","es","Exabot","ExtractorPro","EzineArticlesLinkScanner","Ezooms","FairAd Client","Flaming AttackBot","Foobot","FreeFind","FTRF\\:\\ Friendly","Gaisbot","GetRight/4.2","gigabot","grub","grub-client","Harvest/1.5","Hatena Antenna","hloader","http://www.SearchEngineWorld.com bot","http://www.WebmasterWorld.com bot","HTTP_Request","HTTP_Request2","httplib","humanlinks","hypestat","ia_archiver","ia_archiver/1.6","Indy\\ Library","InfoNaviRobot"]}
{"domain":"atbmarket.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"animeheaven.me","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"nice.org.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["bingbot","*"]}
{"domain":"bellemaison.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"amd.online","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.co.zw","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"flashscore.com.ve","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"roborock.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","adsbot-google"]}
{"domain":"myon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"semprot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["cubestat","*"]}
{"domain":"oceans-nadia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"utoon.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"proxyium.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gismeteo.kz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"webassign.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mykronos.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"myoffice.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot","Yandex"]}
{"domain":"surahquran.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"odysee.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"kupi.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["Mediapartners-Google","MJ12bot","Ezooms","Riddler","*","SeznamBot","AI"]}
{"domain":"finra.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"druni.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot","Adsbot-Google","Mediapartners-Google"]}
{"domain":"flirtify.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"igram.world","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"allhen.online","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"mintmanga.com","fetch_ok":false,"usable":false,"reason":"other","status":402,"error":null,"attempts":[402,402]}
{"domain":"eboblack.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":22,"stanza_tokens":["*","MSNBot","Bingbot","YandexBot","Baiduspider","Sogou","SemrushBot","AhrefsBot","BLEXBot","MJ12bot","008","sistrix","ia_archiver","proximic","SiteExplorer","SemrushBot-SA","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SemrushBot-CT","SemrushBot-BM","SplitSignalBot"]}
{"domain":"optidigital.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ib-game.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"insidehighered.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"happymail.co.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tander.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"imsowiwiwiwiwi.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","TimeoutError"]}
{"domain":"awsdns-40.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"awsdns-07.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"eleadcrm.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"manhwaclan.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"genieesspv.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"rtblab.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"vestiairecollective.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"animesonlinecc.to","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"poste.dz","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"pussyspace.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"kamigame.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hibiny.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":13,"stanza_tokens":["*","Claude-SearchBot","OAI-SearchBot","GPTBot","ClaudeBot","SemrushBot","AhrefsBot","MJ12bot","DotBot","Amazonbot","Bytespider","TikTokSpider","ShapBot"]}
{"domain":"google.co.ck","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"nguoiquansat.vn","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"libramemoria.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":22,"stanza_tokens":["*","PetalBot","BLEXBot","Neevabot","DataForSeoBot","Yandex","AhrefsBot","Sogou","Paqlebot","serpstatbot","TurnitinBot","Buck","SemrushBot","trendictionbot","Amazonbot","anthropic-ai","Bytespider","CCBot","ChatGPT-User","Claude-Web","Google-Extended","GPTBot"]}
{"domain":"aalto.fi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pokepara.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"turfomania.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":["OAI-SearchBot"],"path_restricted":[]}],"stanza_token_count":8,"stanza_tokens":["GPTBot","ChatGPT-User","MSNbot","FacebookBot","ImagesiftBot","AhrefsBot","MJ12bot","weborama-fetcher"]}
{"domain":"sportfilm800.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"w-x.co","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"awsdns-25.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"plsleavemealone.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","TimeoutError"]}
{"domain":"playfootball.games","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"sendinblue.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"choicehotels.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"heritage.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"buffermotivatorsuffice.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"klczy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Googlebot","Slurp","MSNBot","ia_archiver"]}
{"domain":"tibia.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["UND_ERR_CONNECT_TIMEOUT",403]}
{"domain":"betflag.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"physicsandmathstutor.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"k-manga.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","EHOSTUNREACH","TimeoutError"]}
{"domain":"abv.bg","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"umang.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"getredo.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cdnbasket.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"codmwest.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"fdownloader.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"teletype.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"missyusa.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","UNABLE_TO_VERIFY_LEAF_SIGNATURE"]}
{"domain":"lacuerda.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iloveyourweewee.bz","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ECONNRESET"]}
{"domain":"mjt.lu","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bestialitysextaboo.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"alipayobjects.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"dicrotin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"alicesw.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"education.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"jobcan.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lightboxcdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"eluniversal.com.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":35,"stanza_tokens":["*","Googlebot","Googlebot-News","Googlebot-Image","Googlebot-Video","Googlebot-Mobile","Google-Extended","Twitterbot","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","wget","grub-client","k2spider","NPBot","WebReaper"]}
{"domain":"uc.pt","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","CERT_HAS_EXPIRED"]}
{"domain":"tampabay.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":9,"stanza_tokens":["*","Aolbot-News","GPTbot","anthropic-ai","CCBot","OmigiliBot","Bytespider","Diffbot","ClaudeBot"]}
{"domain":"cleanmasterguru.online","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"domaintechnik.at","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tile-api.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sportal.bg","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"whatwg.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["archive.org_bot","ia_archiver","ia_archiver-web.archive.org","*"]}
{"domain":"pjn.gov.ar","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"sunbet.co.za","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"steamunlocked.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"trassir.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"beastiality.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot-Image"]}
{"domain":"nsfwph.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"irancell.ir","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"basalam.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"ns.gov","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"papercut.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xvideos.tube","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"finnair.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","ia_archiver","urlscan.io","Swiftbot","AdsBot-Google","Mediapartners-Google"]}
{"domain":"jkanime.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rugbyrama.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":240,"stanza_tokens":["*","5emeRue","5erue","adequat","adequat-systems","Alexibot","AlvinetSpider","Amazonbot","AmiSoftware","Antenne Hatena","anthropic-ai","ApocalXExplorerBot","Argus","Ask n read","asknread.com","asterias","Augure","auramundi","BackDoorBot/1.0","BizInformation","Black Hole","Bloodhound","Bloomberg","BlowFish/1.0","BotALot","BraveBot","BuiltBotTough","Bullseye/1.0","BunnySlippers","Bytespider","CCBot","Cegbfeieh","ChatGPT-User","OAI-SearchBot","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","Cision","Claude-Web","ClaudeBot","coexel","cohere-ai","ConveraCrawler","CopyRightCheck","Corporama","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","cydralspider","Diffbot","Digimind","DISCo Pump 3.1","DittoSpyder","dotbot","Download Ninja","downloadexpress","EDD","ellisphere","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","eureka","Europresse","Explore","ExtractorPro","FacebookBot","Factiva","Fasterfox","Fetch","Flamingo_SearchEngine","flipboard","Foobot","gammaSpider","Google-Extended","GPTBot","grub-client","Harvest/1.5","hloader","httplib","HTTrack","HTTrack 3.0","humanlinks","ia_archiver","ia_archiver-web.archive.org","Igentia","indexer","InfoNaviRobot","infoseek","JennyBot","Jetbot","JikeSpider","k2spider","Kantar","kbcrawl","Kenjin Spider","Knowings","larbin","leadbox"]}
{"domain":"flashscore.ro","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"ilbe.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-47.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"abhibus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","Baiduspider"]}
{"domain":"metaporn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"dizibox.live","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bynder.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fogaonet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","OAI-SearchBot","ChatGPT-User","PerplexityBot","ClaudeBot","Googlebot","Bingbot"]}
{"domain":"dnevno.hr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sambaporno.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"shop-shimamura.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"wantgoo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"micasino.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":13,"stanza_tokens":["*","Baiduspider","Sogou Spider","360Spider","Yandex","Bytespider","PetalBot","MJ12bot","AhrefsBot","curl","wget","python-requests","HTTrack"]}
{"domain":"pink.rs","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"hasznaltauto.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":21,"stanza_tokens":["*","AdsBot-Google","Googlebot-Image","Mediapartners-Google","trovitBot","kalooga","MJ12bot","proximic","grapeshot","GrapeshotCrawler","WeSEE","WeSEE:Search","admantx","Exabot","SkimBot","Ezooms","AhrefsBot","Mail.RU","Mail.RU_Bot","CrystalSemanticsBot","URLAppendBot"]}
{"domain":"gadsme.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"claveunica.gob.cl","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"detyep.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",404]}
{"domain":"mgkomik.cc","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"baycrews.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"kuruma-news.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rusvideos.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","DuckDuckBot"]}
{"domain":"moppy.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","ia_archiver","*"]}
{"domain":"selfstudys.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":19,"stanza_tokens":["*","Googlebot","Bingbot","WhatsApp","Slackbot","AppEngine-Google","Mediapartners-Google","GPTBot","PerplexityBot","ChatGPT-User","ClaudeBot","anthropic-ai","cohere-ai","DuckAssistBot","Meta-ExternalAgent","OAI-SearchBot","Quora-Bot","PetalBot","DataForSeoBot"]}
{"domain":"bigboss.rodeo","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"awsdns-05.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"ankergames.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"edublogs.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"megapersonals.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xgx.mobi","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"yaoimangaonline.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kidsa-z.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vivastreet.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["PetalBot","*"]}
{"domain":"takarakuji-official.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"aflamk1.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"hopto.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"boat-lifestyle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","adsbot-google"]}
{"domain":"pornogratisdiario.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.com.bz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"uncn.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"klickpin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"creativethemes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nyahentai.one","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"xxvideos.video","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":4,"stanza_tokens":["GPTBot","Amazonbot","ClaudeBot","*"]}
{"domain":"pmkisan.gov.in","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"arionplay.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"alkosto.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","AdsBot-Google","Googlebot-Image"]}
{"domain":"porno-zona.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":26,"stanza_tokens":["*","YandexVideoParser","008","BLEXBot","AhrefsBot","Vagabondo","SEOkicks-Robot","ia_archiver","archive.org_bot","special_archiver","MJ12bot","heritrix","netEstate NE Crawler","sistrix","WBSearchBot","QuerySeekerSpider","proximic","SiteExplorer","SemrushBot","SemrushBot-SA","Mail.Ru","Baiduspider","MSNBot","Bingbot","Slurp","Yandex"]}
{"domain":"smartytouch.co","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nofeu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"steemit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rxnvg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"javgg.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sageone.co.za","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"gfpxw.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"nielsencollections.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pornolandia.xxx","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"meteored.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"litgorod.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ensighten.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"bysex.click","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kunmanga.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"jsonline.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":287,"stanza_tokens":["anthropic-ai","AmazonBot","Claude","Claude-Web","claudeweb","Amzn-SearchBot","Amzn-User","AndiBot","cohere-ai","ClaudeBot","ChatGPT-User","Googlebot-News","GPTBot","CCBot","Google-Extended","DataForSeoBot","Diffbot","Diffbot-user","Kagibot","FacebookBot","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Scrapy","TurnitinBot","Applebot-Extended","Meta-ExternalAgent","meta-externalagent","OAI-SearchBot","Bytespider","Claude-User","Claude-SearchBot","AI2Bot","AhrefsBot","Ai2Bot-Dolma","Applebot","Arquivo-web-crawler","AspiegelBot","AwarioRssBot","AwarioSmartBot","Barkrowler","coccocbot-web","cohere-training-data-crawler","crawler4j","Crawlspace","dotbot","DuckAssistBot","FriendlyCrawler","Google-CloudVertexBot","https://hada.news","https://www.imediaethics.org","iaskspider/2.0","ICC-Crawler","icc-crawler","ICC Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","LivelapBot","MauiBot","Meltwater","Meta-ExternalFetcher","MJ12bot","Mojeek","MojeekBot","MoodleBot","NeticleBot","Netvibes","PanguBot","PetalBot","SeekportBot","Seekr","SemrushBot","SeznamBot","Sidetrade Indexer Bot","Timpibot","Turnitin","VelenPublicWebCrawler","Webzio-Extended","yacy","yacybot","Yandex","YandexAdditional","YandexAdditionalBot","YandexBot","YandexImages","YandexNews","Baiduspider","Baiduspider-image","BaiduSpider-render","Baiduspider-video","Sogou Web Spider","yisouspider"]}
{"domain":"agones.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","ia_archiver"]}
{"domain":"1xlite-08668.world","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"EAI_AGAIN","attempts":["EAI_AGAIN","EAI_AGAIN"]}
{"domain":"alphatv.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mongabay.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"joporn.name","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"krikya11.club","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"9jarocks.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jc-gs.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"awsdns-53.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"vaultdcr.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"google.mg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"xlxx.mobi","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"redtube.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ufrj.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"shalove.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Megalodon","ia_archiver"]}
{"domain":"tanishq.co.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","AdsBot-Google","Googlebot","Googlebot-Image"]}
{"domain":"kriesi.at","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"avidreaders.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","MJ12bot","Yandex","Mediapartners-Google"]}
{"domain":"google.nr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"itorrents-igruha.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"snaptube.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"tmobile-digital.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"dropmms.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"supercp.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"google.com.af","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"resultadofacil.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"dayunlinks.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"jovencitas.gratis","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":33,"stanza_tokens":["*","noxtrumbot","msnbot","Slurp","MSIECrawler","WebCopier","HTTrack","Microsoft.URL.Control","libwww","Orthogaffe","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","SiteSnagger","WebStripper","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","Xenu","larbin","ZyBORG","Download Ninja","wget","grub-client","k2spider","NPBot","WebReaper","Googlebot"]}
{"domain":"szerencsejatek.hu","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"lsgkerala.gov.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"esuteru.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"CERT_HAS_EXPIRED","attempts":["CERT_HAS_EXPIRED","ENOTFOUND"]}
{"domain":"ygles.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"chochox.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sourcefire.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"clave.gob.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"millipiyangoonline.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"leakgallery.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"webaim.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"1x1x5.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"volkskrant.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":29,"stanza_tokens":["Twitterbot","*","GPTBot","ChatGPT-User","CCBot","anthropic-ai","PerplexityBot","Amazonbot","Applebot-Extended","Bytespider","ClaudeBot","Claude-Web","cohere-ai","Diffbot","Google-Extended","Meta-ExternalFetcher","YouBot","DeepSeekBot","MistralAI-User","MistralAI-User/1.0","PetalBot","Timpibot","YandexAdditional","YandexAdditionalBot","YandexBot","YandexRenderBot","Perplexity-User","coccocbot-web","OAI-SearchBot"]}
{"domain":"google.ms","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"21cn.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"mm-cg.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["ia_archiver","archive.org_bot"]}
{"domain":"leolist.cc","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"uafix.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xxxmovies.life","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":9,"stanza_tokens":["Amazonbot","GPTBot","AhrefsBot","Applebot","DataForSeoBot","ByteDance","Bytespider","ClaudeBot","*"]}
{"domain":"eccentricsterilizevastness.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"mhrs.gov.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"iliad.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"24videos.space","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"unitedcinemas.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"naurok.com.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["GoogleBot","*"]}
{"domain":"bigpond.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ERR_TLS_CERT_ALTNAME_INVALID",200]}
{"domain":"appmedia.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tgju.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot","ia_archiver"]}
{"domain":"doofinder.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"autogidas.lt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mycima.cc","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"planetwin365.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"en-thunderscans.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"photoroom.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ecuabet.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"botinok.porn","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":2,"stanza_tokens":["GPTBot","*"]}
{"domain":"perfdrive.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"servicewebly.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bet365.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"clickr.pro","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"nekopoi.care","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"mintmanga.one","fetch_ok":false,"usable":false,"reason":"other","status":402,"error":null,"attempts":[402,402]}
{"domain":"anikai.to","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"uc-share.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"totalcasino.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"buildkite.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"550909.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"naturabuy.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","008","PetalBot","DataForSeoBot","AwarioRssBot","AwarioSmartBot","Sogou","Baiduspider","YisouSpider","AhrefsBot"]}
{"domain":"showcamrips.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"aaronweintraubwantstobecyberpolice.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ENOTFOUND"]}
{"domain":"rfihub.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"belluna.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tameteo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"samehadaku.how","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"aruodas.lt","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.so","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"portaldozacarias.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"josporn.club","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"shopch.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"dorama.land","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"seg-social.gob.es","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"24xxxx.win","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"skilljar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":7,"stanza_tokens":["PetalBot","GPTBot","CCBot","anthropic-ai","Claude-Web","Bytespider","*"]}
{"domain":"search-sprint.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"on3.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["xAI","Grok","grok-ai","xai-bot","*"]}
{"domain":"hover.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"active.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uakino.best","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"deakin.edu.au","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tvazteca.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ondisk.co.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["Yeti"]}
{"domain":"deepika.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"uta-net.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Mediapartners-Google","SemrushBot","proximic","Twitterbot"]}
{"domain":"wort-suchen.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["AmazonAdBot","Mediapartners-Google","grapeshot","YandexBot","SemrushBot","HybridBot","MegaIndex","dotbot","BLEXBot","MJ12bot","proximic","*"]}
{"domain":"webhosting.dk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hostnet.nl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","HuaweiSymantecSpider"]}
{"domain":"dontworry.su","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","ENOTFOUND"]}
{"domain":"coupahost.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"idos.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","grapeshot"]}
{"domain":"liveball.sx","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"coinmastergame.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"enterprise.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":25,"stanza_tokens":["AndiBot","Applebot","Applebot-Extended","bingbot","Bytespider","CCBot","ChatGPT-User","Claude-SearchBot","Claude-User","ClaudeBot","DuckAssistBot","ExaBot","FirecrawlAgent","Google-Agent","Google-Extended","Googlebot","GPTBot","meta-externalagent","meta-externalfetcher","meta-webindexer","Perplexity-User","PerplexityBot","PhindBot","YouBot","*"]}
{"domain":"smartinmate.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ECONNRESET",404]}
{"domain":"avtub.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mlookalporno.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sch.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["GPTBot","Amazonbot","ClaudeBot","Bytespider","CCBot","PerplexityBot","*"]}
{"domain":"languagetoolplus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pixocial.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"janeapp.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"javsek.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"tracookiepixel.xyz","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"google.co.mz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"gapgpt.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","GPTBot","ChatGPT-User","anthropic-ai","Claude-Web"]}
{"domain":"skmada.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"jav.sb","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hely.tech","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"slon4.at","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"nontonanimeid.boats","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"egov.uz","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"boatrace.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bids.ws","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"supportview.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","TimeoutError"]}
{"domain":"allahabadhighcourt.in","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"fonts.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"directv.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"mshome.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"pearson-intl.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"arabnews.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"google.st","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"simonandschuster.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jobbmintatv.pro","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xn----ztbcbceder.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ridigital.org.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"scba.gov.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot","*"]}
{"domain":"proteantech.in","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"skyroom.online","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","GPTBot","ChatGPT-User","ClaudeBot","PerplexityBot","Google-Extended"]}
{"domain":"awsdns-12.co.uk","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"wordunscrambler.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"watchhentai.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"brawlstarsgame.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"allianz.de","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"jenkins.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",404]}
{"domain":"sorrisi.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"linuxfoundation.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pornobrasil.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pcmax.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["ia_archiver","Baiduspider","BaiduImagespider","yodaobot","Yeti","NaverBot","Googlebot","*"]}
{"domain":"apornvideo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"browserstack.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.td","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"pointtown.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["AdsBot-Google","Mediapartners-Google","*"]}
{"domain":"eduvulcan.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"arabxn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Copyleaks","Plagiarism-Checker"]}
{"domain":"desikahani2.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"kemdiktisaintek.go.id","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",404]}
{"domain":"gegen-hartz.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"espn.com.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":11,"stanza_tokens":["claritybot","GPTBot","Google-Extended","CCBot","ChatGPT-User","anthropic-ai","Omgilibot","Omgili","FacebookBot","Bytespider","*"]}
{"domain":"kunlunle.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"scrabblewordfinder.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"provenexpert.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yoomoney.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"tensorflow.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"infura-ipfs.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"deutschlandfunk.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"saya.pk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","adsbot-google"]}
{"domain":"sassa.gov.za","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"my-control-panel.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nagad88f.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"airee.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"winline.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"csc.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"epicgamescdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sps-system.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mailin.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Twitterbot","facebookexternalhit"]}
{"domain":"prefeitura.sp.gov.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lesoir.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":137,"stanza_tokens":["Mediapartners-Google","Googlebot","Googlebot-Image","Googlebot-Mobile","Googlebot-News","Googlebot-Video","Adsbot-Google","Twitterbot","Applebot","Bingbot","SiteAuditBot","Publication-Access-for-Facebook","facebookexternalhit","Flipboard","FlipboardProxy","upday","AhrefsBot","AhrefsSiteAudit","5emeRue","5erue","adequat","adequat-systems","AmiSoftware","AwarioRssBot","AwarioSmartBot","Argus","Ask n read","asknread.com","Augure","auramundi","Bloodhound","Cision","coexel","ConveraCrawler","Corporama","cydralspider","Digimind","Download Ninja","downloadexpress","EDD","ellisphere","eureka","Europresse","Explore","Factiva","Fasterfox","Fetch","gammaSpider","grub-client","HTTrack","ia_archiver","ia_archiver-web.archive.org","indexer","infoseek","Jetbot","k2spider","Kantar","kbcrawl","Knowings","larbin","leadbox","libwww","linkfluence","linko","manageo","mediacompil","Meltwater","mention","Moreover","MSIECrawler","mytwip","newscan-online","NewsNow","Newzbin","NPBot","ObjectsSearch","Offline Explorer","opinion-tracker","Pimptrain","proxem","QuepasaCreep","Qwam content intelligence","Raven","readability.com","scoop.it","score3","Sindup","sitecheck.internetseer.com","SiteSnagger","spotter","Synthesio","Talkwater","Teleport","TeleportPro","trendeo","trendybuzz","TunitinBot","TurnitinBot","up2news","vecteurplus"]}
{"domain":"madisoft.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"webofcomics.top","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":192,"stanza_tokens":["*","Yandex","Googlebot","Java","NjuiceBot","Gigabot","Scrapy","Baiduspider","SeznamBot","crawler","JS-Kit","HybridBot","Voyager","PostRank","DomainCrawler","SemrushBot","MegaIndex.ru","ltx71","SurveyBot","AhrefsBot","Exabot","Aport","CCBot","DotBot","ia_archiver","larbin","Butterfly","libwww","bingbot","Wget","SWeb","LinkExchanger","Soup","GrapeshotCrawler","WordPress","DnyzBot","spbot","DeuSu","MLBot","InternetSeer","BUbiNG","FairShare","Yeti","Birubot","YottosBot","Linguee","Ezooms","lwp-trivial","Purebot","kmSearchBot","SiteBot","CamontSpider","ptd-crawler","HTTrack","suggybot","ttCrawler","Nutch","msnbot","msnbot-media","Slurp","Zeus","Abonti","aggregator","asterias","BDCbot","BLEXBot","BuiltBotTough","Bullseye","BunnySlippers","Cegbfeieh","CheeseBot","CherryPicker","coccoc","CopyRightCheck","cosmos","Crescent","discobot","DittoSpyder","Download Ninja","EasouSpider","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","ExtractorPro","Fasterfox","FeedBooster","Foobot","Genieo","Harvest","hloader","httplib","humanlinks","ieautodiscovery","Incutio","InfoNaviRobot","IstellaBot","JamesBOT","JennyBot","k2spider"]}
{"domain":"bytedns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mykadri.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uv.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rax.io","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"ENOTFOUND"]}
{"domain":"tkc.co.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"3gpking.name","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",403]}
{"domain":"rtcxyz.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"adstk.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"arabsex1.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"ymcdn.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",404]}
{"domain":"monsnode.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"drecom.jp","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"bamper.by","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"escortbabylon.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"semana.es","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"lucky3357.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"destacamos.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"toptoon.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","MJ12bot","J12bot","coccocbot","Googlebot","Yeti"]}
{"domain":"99xo4.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":39,"stanza_tokens":["Amazonbot","Anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","Cohere-ai","DataForSeoBot","Google-Extended","GPTBot","ImagesiftBot","Magpie-crawler","Omgili","Omgilibot","Peer39_crawler","Peer39_crawler/1.0","PerplexityBot","YouBot","Googlebot","Googlebot-Image","Googlebot-News","Googlebot-Video","AdSense","AdsBot-Google","AdsBot","Google-InspectionTool","APIs-Google","Feedfetcher-Google","Google-NotebookLM","facebookexternalhit","Facebot","meta-externalagent","Facebook","Meta","*"]}
{"domain":"flashscore.pe","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"mastiraja.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"net-empregos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","Yandex","MJ12bot","AhrefsBot","grapeshot","VelenPublicWebCrawler","CriteoBot/0.1","PetalBot"]}
{"domain":"onclouddns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"xvideosbr.blog","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lolpol2.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"afun.mx","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"karger.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["008","*"]}
{"domain":"google.cf","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"pkok11.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":["OAI-SearchBot"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":29,"stanza_tokens":["AI2Bot","Amazonbot","anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","DataForSeoBot","Diffbot","DuckAssistBot","FacebookBot","Google-Extended","GPTBot","ImagesiftBot","Magpie-crawler","Omgili","Omgilibot","Peer39_crawler","Peer39_crawler/1.0","PerplexityBot","PetalBot","Timpibot","YouBot","*"]}
{"domain":"soap2day.day","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"thalesgroup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","meta-externalagent"]}
{"domain":"meripehchaan.gov.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.ws","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"wishlink.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"jin115.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot-Mobile"]}
{"domain":"komiku.asia","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"familyclub.jp","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"odibets.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"mydesi2.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"investors.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":25,"stanza_tokens":["*","AmazonAdBot","Google-Extended","BLP_bbot","BLP_bbot/0.1","CCBot","anthropic-ai","cohere-ai","ia_archiver","omgili","omgilibot","MJ12bot","PiplBot","Bytespider","ClaudeBot","Claude-Web","FacebookBot","magpie-crawler","NewsNow","news-please","PerplexityBot","Perplexity-ai","Meta-ExternalAgent","Meta-ExternalFetcher","Applebot-Extended"]}
{"domain":"cloudflarestream.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"arm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":23,"stanza_tokens":["anthropic-ai","GPTBot","ClaudeBot","Claude-web","CCBot","Google-Extended","Amazonbot","Applebot","Bingbot","ChatGPT-User","Bytespider","PerplexityBot","Sogou","CoveoEnterpriseSearch","*","AhrefsBot","YandexBot","MegaIndex.ru","SemrushBot","Qwantify/Bleriot","DotBot","MJ12bot","SEOkicks"]}
{"domain":"topcinemaa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"learncbse.in","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"manhwa-online.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"hsex.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":11,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","Googlebot-Image"]}
{"domain":"ticktick.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vidmate-official.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"telemagazyn.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":9,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"gsxt.gov.cn","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"truper.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["Yandex","sogou","TurnitinBot","VeBot","Cliqzbot","BLEXBot","AhrefsBot","Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)","msnbot","Bingbot","MSNBot","MSNBot-Media","AdIdxBot","BingPreview","*"]}
{"domain":"survicate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"q10.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"musicaldown.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ddizi.im","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.com.fj","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"luvyaa.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"superproxy.io","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"3xyaoi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"cv666.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":["OAI-SearchBot"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]},{"family":"perplexity","blocked":["PerplexityBot"],"open":["Perplexity-User"],"path_restricted":[]}],"stanza_token_count":29,"stanza_tokens":["AI2Bot","Amazonbot","anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","DataForSeoBot","Diffbot","DuckAssistBot","FacebookBot","Google-Extended","GPTBot","ImagesiftBot","Magpie-crawler","Omgili","Omgilibot","Peer39_crawler","Peer39_crawler/1.0","PerplexityBot","PetalBot","Timpibot","YouBot","*"]}
{"domain":"strmd.st","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mergesigma.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"arenaplus.ph","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"taptapnetworks.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","attempts":["ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR","ENOTFOUND"]}
{"domain":"gs.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"trustburn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"findnovel.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"pi7.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xossipy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["ia_archiver","*"]}
{"domain":"medex.com.bd","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-48.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kpn.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"justindianporn.info","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":22,"stanza_tokens":["*","008","BLEXBot","AhrefsBot","Vagabondo","SEOkicks-Robot","ia_archiver","archive.org_bot","special_archiver","MJ12bot","heritrix","netEstate NE Crawler","sistrix","WBSearchBot","QuerySeekerSpider","proximic","SiteExplorer","SemrushBot","SemrushBot-SA","Baiduspider","MSNBot","Bingbot"]}
{"domain":"nan-net.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"anime-sama.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-00.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"mxtoolbox.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","AhrefsBot","dotbot","meta-externalagent"]}
{"domain":"navdmp.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"javrate.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dhlottery.co.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sellatuparley.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zeturf.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pomponik.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Mediapartners-Google"]}
{"domain":"pebblego.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tse.jus.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"aniagotuje.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":9,"stanza_tokens":["GPTBot","Google-Extended","ClaudeBot","Bytespider","CCBot","Applebot-Extended","FacebookBot","omgili","*"]}
{"domain":"nelomanga.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","DMCAAgent","CopyrightCrawler","AntiPiracyBot","008","Slurp","goo-crawler","RakutenBot","Teoma","Baiduspider","Sogou Spider","360Spider","YisouSpider","ia_archiver","MusoBot","Link-BusterBot"]}
{"domain":"filimo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rednotecdn.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"lettuceclub.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"whois.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","AhrefsBot","dotbot","MJ12bot","SemrushBot","SemrushBot-SA"]}
{"domain":"cityu.edu.hk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"blockaway.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nebo.mobi","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"search-io.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"fireeye.com","fetch_ok":false,"usable":false,"reason":"other","status":530,"error":null,"attempts":[530,530,530]}
{"domain":"dnews.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"johnpyeauctions.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","TermlyBot"]}
{"domain":"granola.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"redislabs.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"eropuru.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lidl.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"detroitnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":287,"stanza_tokens":["anthropic-ai","AmazonBot","Claude","Claude-Web","claudeweb","Amzn-SearchBot","Amzn-User","AndiBot","cohere-ai","ClaudeBot","ChatGPT-User","Googlebot-News","GPTBot","CCBot","Google-Extended","DataForSeoBot","Diffbot","Diffbot-user","Kagibot","FacebookBot","magpie-crawler","NewsNow","news-please","omgili","omgilibot","peer39_crawler","peer39_crawler/1.0","PerplexityBot","Scrapy","TurnitinBot","Applebot-Extended","Meta-ExternalAgent","meta-externalagent","OAI-SearchBot","Bytespider","Claude-User","Claude-SearchBot","AI2Bot","AhrefsBot","Ai2Bot-Dolma","Applebot","Arquivo-web-crawler","AspiegelBot","AwarioRssBot","AwarioSmartBot","Barkrowler","coccocbot-web","cohere-training-data-crawler","crawler4j","Crawlspace","dotbot","DuckAssistBot","FriendlyCrawler","Google-CloudVertexBot","https://hada.news","https://www.imediaethics.org","iaskspider/2.0","ICC-Crawler","icc-crawler","ICC Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","JenkersBot","Kangaroo Bot","LivelapBot","MauiBot","Meltwater","Meta-ExternalFetcher","MJ12bot","Mojeek","MojeekBot","MoodleBot","NeticleBot","Netvibes","PanguBot","PetalBot","SeekportBot","Seekr","SemrushBot","SeznamBot","Sidetrade Indexer Bot","Timpibot","Turnitin","VelenPublicWebCrawler","Webzio-Extended","yacy","yacybot","Yandex","YandexAdditional","YandexAdditionalBot","YandexBot","YandexImages","YandexNews","Baiduspider","Baiduspider-image","BaiduSpider-render","Baiduspider-video","Sogou Web Spider","yisouspider"]}
{"domain":"roeye.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"priceoye.pk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":436,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","Alexibot","Apache-HttpClient","Aqua_Products","archive.org_bot","asterias","b2w/0.1","BackDoorBot","BackDoorBot/1.0","Baiduspider","Baiduspider-image","Baiduspider-video","Black.Hole","BlackWidow","BLEXBot","BlowFish","BlowFish/1.0","Bookmark search tool","Bot mailto:craftbot@yahoo.com","BotALot","BotRightHere","BPImageWalker/2.0","BuiltBotTough","Bullseye","Bullseye/1.0","BunnySlippers","Cegbfeieh","CheeseBot","CherryPicker","CherryPickerElite/1.0","CherryPickerSE/1.0","ChinaClaw","Cliqzbot","complex_network_group","Copernic","CopyRightCheck","cosmos","Crescent","Crescent Internet ToolPak HTTP OLE Control v.1.0","Custo","DISCo","DISCo Pump 3.0","DISCo Pump 3.2","DISCoFinder","DittoSpyder","DoCoMo","DotBot","Download Demon","Download Demon/3.2.0.8","Download Demon/3.5.0.11","eCatch","eCatch/3.0","EirGrabber","EmailCollector","EmailSiphon","EmailWolf","EroCrawler","Experibot_v1 [bit.ly","Express WebPictures","Express WebPictures (www.express-soft.com)","ExtractorPro","EyeNetIE","FairAd Client","Flaming AttackBot","FlashGet","FlashGet WebWasher 3.2","Foobot","FrontPage","FrontPage [NC,OR]","Gaisbot","GetRight","GetRight/2.11","GetRight/3.1","GetRight/3.2","GetRight/3.3","GetRight/3.3.3","GetRight/3.3.4","GetRight/4.0.0","GetRight/4.1.0","GetRight/4.1.1","GetRight/4.1.2","GetRight/4.2","GetRight/4.2b (Portuguxeas)","GetRight/4.2c","GetRight/4.3","GetRight/4.5","GetRight/4.5a","GetRight/4.5b","GetRight/4.5b1","GetRight/4.5b2","GetRight/4.5b3"]}
{"domain":"google.cv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"myhentaigallery.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"estama.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"riyasewana.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["DotBot","AhrefsBot","SemrushBot","MJ12bot","IASBot","*"]}
{"domain":"umingle.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"siftscience.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","GPTBot","ChatGPT-User","PerplexityBot","ClaudeBot","Google-Extended","Amazonbot","Applebot-Extended","CCBot"]}
{"domain":"gays-cruising.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot-Image","*"]}
{"domain":"vlxx.moi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"leckerschmecker.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":103,"stanza_tokens":["*","AddSearchBot","AI2Bot","Ai2Bot-Dolma","aiHitBot","amazon-kendra-","Amazonbot","Andibot","Anomura","anthropic-ai","Applebot","Applebot-Extended","Awario","Baiduspider","bedrockbot","bigsur.ai","Bravebot","Brightbot 1.0","Bytespider","CCBot","GPTBot","ChatGPT Agent","ChatGPT-User","ClaudeBot","Claude-Web","Claude-SearchBot","Claude-User","Cloudflare-AutoRAG","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Datenbank Crawler","DeepSeekBot","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","EvilAISample","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","Google-Firebase","Google-NotebookLM","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","iaskspider/2.0","ia_archiver","IbouBot","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","LinerBot","magpie-crawler","meta-webindexer","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NovaAct","OAI-SearchBot","omgili","omgilibot","OpenAI","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","ShapBot","Sidetrade indexer bot","TerraCotta","Thinkbot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","wpbot","YaK"]}
{"domain":"templatemonster.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"gogoroyal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"solitairegrandharvest.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jianshu.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"hentaila.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mcw-casino.games","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"muumuu-domain.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","BLEXBot","meta-externalagent","MJ12bot","SemrushBot","serpstatbot","AhrefsBot","Baiduspider","bingbot"]}
{"domain":"themoscowtimes.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"geoipcheck.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"wikicv.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"uipath.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","Googlebot","AdsBot-Google","Googlebot-Image"]}
{"domain":"hwclouds-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"cnnamador.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":18,"stanza_tokens":["Googlebot","Bingbot","Google-Extended","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","GPTBot","MJ12bot","DotBot","PetalBot","SemrushBot","AhrefsBot","DataForSeoBot","Crawler4j","Scrapy","*"]}
{"domain":"avdbs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["MJ12bot","YandexBot","DataForSeoBot","Twitterbot","facebookexternalhit","bingbot","msnbot","GPTBot","Applebot","*"]}
{"domain":"tv9telugu.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Baiduspider"]}
{"domain":"groupme.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"a1.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"vodafone.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"evotor.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Yandex","Googlebot"]}
{"domain":"truyenfull.vision","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kantimehealth.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"123-hds.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"rosszlanyok.hu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot-Image"]}
{"domain":"novelfull.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"educacion.gob.es","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UND_ERR_SOCKET","attempts":["UND_ERR_SOCKET","UND_ERR_SOCKET"]}
{"domain":"luticlip.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AdsBot"]}
{"domain":"i993.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"linkmake.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"nameserverservice.de","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"sinoptik.bg","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"adglare.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ENOTFOUND"]}
{"domain":"sushida.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"google.li","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"placardefutebol.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"meteored.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-18.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kingoftime.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"snaptik.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kqbd.mobi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"buumal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.ht","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"litboom.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"zapto.org","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"exlibrisgroup.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hkirc.net.hk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ERR_SSL_TLSV1_UNRECOGNIZED_NAME"]}
{"domain":"unipd.it","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"huffingtonpost.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":211,"stanza_tokens":["*","Meltwater","Cision","Talkwater","Jetbot","kbcrawl","Newzbin","Qwam content intelligence","flipboard","Youmag","Synthesio","trendybuzz","scoop.it","linkfluence","grub-client","ia_archiver-web.archive.org","k2spider","libwww","wget","5erue","adequat","adequat-systems","coexel","leadbox","mention","mytwip","opinion-tracker","proxem","score3","vecteurplus","verticalsearch","vsw","winello","Fetch","infoseek","MSIECrawler","Offline Explorer","sitecheck.internetseer.com","Teleport","TeleportPro","WebCopier","WebStripper","Zealbot","asknread.com","omgilibot","omgili","CCBot","Google-Extended","PerplexityBot","FacebookBot","anthropic-ai","Claude-Web","ClaudeBot","Applebot-Extended","Webzio-Extended","Meta-ExternalFetcher","Amazonbot","Timpibot","AI2Bot","Applebot","cohere-training-data-crawler","DuckAssistBot","Kangaroo Bot","Meta-ExternalAgent","PanguBot","5emeRue","ACQUIRE MEDIA","ACTIV Financial (CME Group)","AlphaSense","AmiSoftware","archive.org_bot","Archive-It","ArgClrInt","Ask n read","Augure","auramundi","AwarioRssBot","AwarioSmartBot","Barchart.com","BattleFin","Bernin IT","Blackboard Safeassign","BLP_bbot","bluematrix","Brandwatch","Briefcase.news","Buck","Bytespider","CCBo","CikisiBot","Coexel","cohere-ai","Comtex News Network","ConveraCrawler","Copyright Licensing Agency","Corporama","D&B Hoovers","Data Expression","Data Observer","Dataminr"]}
{"domain":"brstej.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Googlebot","Googlebot-Image","Googlebot-Mobile","Googlebot-Video","Mediapartners-Google"]}
{"domain":"freevideo.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"xy8777.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_GET_ISSUER_CERT_LOCALLY","attempts":["UNABLE_TO_GET_ISSUER_CERT_LOCALLY","UNABLE_TO_GET_ISSUER_CERT_LOCALLY"]}
{"domain":"watchanimeworld.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"avkuy.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pawwalls.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"awsdns-44.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"census.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"inews.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":103,"stanza_tokens":["*","archive-it.org","archive.org_bot","ArchiveBot","Arquivo-web-crawler","Authory","babel","bl.uk_ldfc_bot","bl.uk_ldfc_renderbot","bne.es_bot","bnf.fr_bot","ecoresearch","ecoresearchCrawler","Europarchive.org","kb.dk_bot","Kulturarw3","LivelapBot","mediacloud","mirrorweb","MSIECrawler","nationalarchives.gov.uk","netarkivet.dk","netEstate NE Crawler","Nicecrawler","nl-israel_iaharvester2026","nlnbot","nlncrawler","PageFreezer","SafeDNS","SmarshBot","special_archiver","tkl.iis.u-tokyo.ac.jp","TurnitinBot","webarchiv.cz","WordPress.com mShots","XY-Archive-Compliance-Archiver","XY-Archive-Compliance-Crawler","AhrefsBot","AI2Bot","Ai2Bot-Dolma","Amazonbot","amazon-kendra-","anthropic-ai","Applebot-Extended","bedrockbot","Bytespider","CloudflareBrowserRenderingCrawler","CCBot","ChatGLM-Spider","ChatGPT-User","ClaudeBot","Claude-SearchBot","Claude-User","Claude-Web","cohere-ai","Cotoyogi","DeepSeekBot","Diffbot","DuckAssistBot","EchoboxBot","FacebookBot","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","Grok","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","KunatoCrawler","LinerBot","Meltwater","Meta-ExternalAgent","Meta-ExternalFetcher","MistralAI-User","OAI-Operator","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","Perplexity-User","PetalBot","QualifiedBot","Scrapy","Seekr","Sidetrade indexer bot","TaraGroup Intelligent Bot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended"]}
{"domain":"vervegroup.cloud","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"dropgame.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ntp.org.cn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"ichancy.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"pje.jus.br","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"goraggio.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UNABLE_TO_VERIFY_LEAF_SIGNATURE","attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE","TimeoutError"]}
{"domain":"lto.gov.ph","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"iptv2022.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"futurelearn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"playsport.cc","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"timesnowhindi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":18,"stanza_tokens":["*","Baiduspider","Yandex","DataForSeoBot","Screaming Frog SEO Spider","SemrushBot","Meta-ExternalAgent","Applebot","Amazonbot","BLEXBot","PetalBot","DotBot","Bytespider","ClaudeBot","GPTBot","ChatGPT-User","OAI-SearchBot","AmazonAdBot"]}
{"domain":"shad.ir","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"techspot.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Mediapartners-Google","AmazonAdBot","*"]}
{"domain":"lpsnmedia.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"vercomicsporno.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"avaeduc.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"juegosonce.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cimalight.co","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"minhngoc.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError",200]}
{"domain":"bizkub.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"jamb.gov.ng","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"xskt.com.vn","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"albumaty.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mcmaster.ca","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"sarkariresult.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"omni.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"jcplatform.dev","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"google.co.tz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"fngml.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":3,"stanza_tokens":["*","GPTBot","ChatGPT-User"]}
{"domain":"xnalgas.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"turf-fr.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"paygrid.world","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"thscore3.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"crystalbet.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"espn.com.do","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]}],"stanza_token_count":11,"stanza_tokens":["claritybot","GPTBot","Google-Extended","CCBot","ChatGPT-User","anthropic-ai","Omgilibot","Omgili","FacebookBot","Bytespider","*"]}
{"domain":"sengokuixa.jp","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"nepremicnine.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"viralkand.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"freedot.info","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"lpga.or.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"masstamilan.dev","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*","SemrushBot","MJ12bot","BLEXBot"]}
{"domain":"ssrchat.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"p6sai.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ECONNREFUSED","TimeoutError"]}
{"domain":"ktrmr.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"illuminateed.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bidgx.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"rtb-oveeo.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",404]}
{"domain":"windowscentral.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":15,"stanza_tokens":["*","AmazonAdBot","bytespider","mistralai","cohere","ai2bot","youbot","omgili","diffbot","kangaroo","img2dataset","amazonbot","amazon-qbusiness","meta-externalagent","meta-webindexer"]}
{"domain":"portal6.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uakinogo.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","BLEXBot","MJ12bot","SemrushBot","PetalBot","CriteoBot","proximic","grapeshot","ia_archiver"]}
{"domain":"xnxxarab.to","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"todorelatos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":17,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","archive.org_bot","BoardReader","PerplexityBot","FacebookBot","OmgiliBot","anthropic-ai","cohere-ai"]}
{"domain":"huyamba.name","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","CERT_HAS_EXPIRED"]}
{"domain":"google.bt","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"google.sm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"aladin.co.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","QuerySeekerSpider ( http://queryseeker.com/bot.html )","AhrefsBot","MJ12bot","SemrushBot","Baiduspider","Ezooms","YandexBot","ltx71","zgrab/0.x"]}
{"domain":"mcdvoice.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"premierbet.co.ao","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cun.edu.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":34,"stanza_tokens":["*","Googlebot","AhrefsBot","noxtrumbot","msnbot","Slurp","MSIECrawler","WebCopier","HTTrack","Microsoft.URL.Control","libwww","Orthogaffe","UbiCrawler","DOC","Zao","sitecheck.internetseer.com","Zealbot","SiteSnagger","WebStripper","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","Xenu","larbin","ZyBORG","Download Ninja","wget","grub-client","k2spider","NPBot","WebReaper"]}
{"domain":"888scoreonline.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"e-jumbo.gr","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"jecontacte.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["ia_archiver"]}
{"domain":"wfxs.tw","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":27,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","Googlebot","Bingbot","Slurp","Teoma","ia_archiver","twiceler","MSNBot","Scrubby","Robozilla","Gigabot","googlebot-image","googlebot-mobile","yahoo-mmcrawler","yahoo-blogs/v3.9","psbot","OpenAIOpenAI","Applebot"]}
{"domain":"krx18.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"anichin.cafe","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"best-hit.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["Amazonbot"]}
{"domain":"zone-turf.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":4,"stanza_tokens":["*<br/>","GPTBot<br/>","ChatGPT-User<br/>","CCBot<br/>"]}
{"domain":"olympusbiblioteca.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"canlidoviz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"idolfap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"javmix.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kamareels2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"spotify.net","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"metroopinion.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"abozeb.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"yakkun.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":18,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","ChatGPT-User","SemrushBot","AhrefsBot","PetalBot","BLEXBot","Linguee","MJ12bot","DotBot"]}
{"domain":"jdwel.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.cg","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"evvelcevap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nature.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ovo.id","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"escortsandbabes.com.au","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"bet365.bet.ar","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mobiledokan.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hotpic.cc","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"zefoy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"indianoil.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"egramswaraj.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"wfp.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gay0day.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"shop-pro.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"twinsfun.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"etrain.info","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"lel.asia","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"bkn.go.id","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"qvc.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"anonymised.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":31,"stanza_tokens":["Googlebot","Bingbot","Applebot","DuckDuckBot","OAI-SearchBot","ChatGPT-User","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","DuckAssistBot","GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Amazonbot","cohere-ai","Bytespider","meta-externalagent","ImageSiftBot","Diffbot","Omgilibot","Omgili","PetalBot","MJ12bot","SemrushBot","AhrefsBot","Claude-Web","anthropic-ai","*"]}
{"domain":"andersnoren.se","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ros.ie","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"stps.gob.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"taplink.cc","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"arbada.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"p7cloud.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"arpanet.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"delta-executor.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"oaed.gr","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"corppn.ru","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"banesconline.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"fortunebusinessinsights.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"com.se","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNRESET","attempts":["ECONNRESET","TimeoutError"]}
{"domain":"pornozak.shop","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"hyperliquid.xyz","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"dnsnode.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"ihk.de","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"cdnhwchcg02.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"bokepnoz.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"babycenter.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":94,"stanza_tokens":["008","AddSearchBot","AI2Bot","Ai2Bot-Dolma","Ai2Bot-DeepResearchEval","Amazonbot","AmazonBuyForMe","Anchor Browser","Anomura","anthropic-ai","atlassian-bot","AwarioSmartBot","Applebot","Applebot-extended","bigsur.ai","Brandwatch","Bravebot","Bytespider","CCBot","Channel3Bot","ChatGLM-Spider","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","Cloudflare-AutoRAG","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawl4AI","Datenbank-Crawler","DDM-DCipher","DeepSeekBot","Devin","Diffbot","DuckAssistBot","Echobot Bot","FacebookBot","Factset_spyderbot","GPTBot","HTTrack","iAsk","iaskspider","ICC-Crawler","ImageSiftBot","imageSpider","Kangaroo-Bot","KlaviyoAIBot","Kunato","laion-huggingface-processor","LinerBot","LCC","LinkupBot","Manus-User","meta-externalagent","meta-externalfetcher","meta-webindexer","mistral.ai","MistralAI-User","netEstate Imprint Crawler","NovaAct","Novellum","Nutch","OAI-SearchBot","Offline Explorer","omgili","omgilibot","PanguBot","PeopleInc-DCipher-Scraper/1.1.0","Perplexity-User","PerplexityBot","PetalBot","Poggio-Citations","QualifiedBot","SBIntuitionsBot","Scrapy","SeekrBot","SemrushBot-OCOB","SemrushBotSwa","ShapBot","Spider","TavilyBot","Timpibot","TwinAgent","Velen Crawler","VelenPublicWebCrawler","Webzio-Extended","Wrtn","YisouSpider","YouBot","ZanistaBot","*","AdsBot-Google"]}
{"domain":"netafraz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dord.gov.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"novavexis.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"futbol-11.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"buchstaben.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"livescore.mobi","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":8,"stanza_tokens":["SleepBot","ClaudeBot","GPTBot","Amazonbot","meta-externalagent","Baiduspider","AhrefsBot","*"]}
{"domain":"animeshentai.biz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"microworkers.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"9anime.org.lv","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"crickexnow.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"fairyanime.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"juan365.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"go-manga.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"andpad.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tele2.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"loteriasdominicanas.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"tjrs.jus.br","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"uzporno.site","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"webdramaturkey2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"westerndigital.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["wdcbot","OAI-SearchBot","*"]}
{"domain":"prajavani.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":17,"stanza_tokens":["Mediapartners-Google","SeekportBot","Yandex","GPTBot","GPTCrawler","ChatGPT-User","OAI-SearchBot","ClaudeBot","anthropic-ai","Google-Extended","CCBot","PerplexityBot","Perplexity-User","Bytespider","Bytespider-webpage","Amazonbot","*"]}
{"domain":"modaselvim.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":43,"stanza_tokens":["*","W3C-checklink","W3C_Validator","W3C_CSS_Validator","Baiduspider","archive.org_bot","ia_archiver","FavIconizer","AhrefsBot","A6-Indexer/1.0","butterfly","Exabot","grapeshot","Mail.Ru","nutch","pingdom","proximic","Python","SearchmetricsBot","SeznamBot","Sogou web spider","sogou spider","Sogou","YoudaoBot","Sosospider","spbot","Wget","yie8","MJ12bot","360Spider","TurnitinBot","BLEXBot","hivaBot","NerdyBot","updown_tester","AndroidDownloadManager","voltron","Xenu's Link Sleuth 1.1c","SemrushBot","SemrushBot-SA","oBot","Riddler","Mail.RU_Bot"]}
{"domain":"sexneka.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"crummydevioussucculent.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"61yes.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"studying.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":8,"stanza_tokens":["*","MJ12bot","AhrefsBot","BLEXBot","Yandex","baiduspider","SemrushBot","BingBot"]}
{"domain":"oray.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tubepussy.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"cimanow.cc","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"zi-scripts.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"sproutgigs.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"pex.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"baomoi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uaserials.my","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["BLEXBot","MJ12bot","CriteoBot/0.1","SemrushBot","PetalBot","proximic","grapeshot","ia_archiver","AhrefsBot","*"]}
{"domain":"gamemania.co.ke","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"simontokx.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"uukanshu.cc","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"state.co.us","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"gobid.co.za","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"surabaya.go.id","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"citapreviadnie.es","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"awsdns-56.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"filesdl.top","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"miku-doujin.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tpmn.co.kr","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"uib.no","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xleche.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ketogo.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ixdzs.tw","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"css-tricks.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":12,"stanza_tokens":["*","GPTBot","ChatGPT-User","CCBot","anthropic-ai","ClaudeBot","Google-Extended","PerplexityBot","Bytespider","SemrushBot","AhrefsBot","DotBot"]}
{"domain":"cpbl.com.tw","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"naijaprey.tv","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-46.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"garotocomlocal.com.br","fetch_ok":false,"usable":false,"reason":"other","status":405,"error":null,"attempts":[405,"ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"fortiddns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"minsa.gob.pe","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ECONNRESET",403]}
{"domain":"market-qx.trade","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"betting.co.zw","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"dochord.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"clashofclans-layouts.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"services.gc.ca","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"kurnik.pl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"everlastting.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"gorodok.bz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"ddd-smart.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"d-deltanet.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"turkcell.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Googlebot","AdsBot-Google","AdsBot-Google-Mobile","Googlebot-Image"]}
{"domain":"obramat.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AdsBot-Google"]}
{"domain":"analog.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"youngle.tech","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"hsforms.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"elephantbet.co.mz","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tdgall.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":6,"stanza_tokens":["*","Yeti","Googlebot","ZumBot","SemrushBot","YandexBot"]}
{"domain":"plumenet.io","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"apl-alexa.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"nu.or.id","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"desibf.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"usabilla.com","fetch_ok":false,"usable":false,"reason":"other","status":503,"error":null,"attempts":["TimeoutError",503,503]}
{"domain":"decathlon.de","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sosalkino.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"solopornoitaliani.xxx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"baji999.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"vide-greniers.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":24,"stanza_tokens":["*","Googlebot","Googlebot-News","Googlebot-Image","Googlebot-Video","Storebot-Google","Google-InspectionTool","GoogleOther","Google-Extended","AdsBot-Google-Mobile","AdsBot-Google","Mediapartners-Google","Bingbot","MSNBot","MSNBot-Media","AdIdxBot","BingPreview","AmazonAdBot","Slurp","DuckDuckBot","Applebot","FacebookExternalHit","Twitterbot","LinkedInBot"]}
{"domain":"24smi.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tamilgun.now","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"indiadafa.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","MJ12bot"]}
{"domain":"tapatalk.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"adorocinema.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["AmazonAdBot","*","seekbot","Exabot","Slurp","Cliqzbot","AI2Bot","ChatGLM","ChatGLM-spider","cohere-training-data-crawler","Diffbot","OmigiliBot","PetalBot","YouBot"]}
{"domain":"pnc.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"blahlivedsnowdrop.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Googlebot"]}
{"domain":"myshopline.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[404,403]}
{"domain":"goseries4k.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"xxxindianstories.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"678bd44.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":39,"stanza_tokens":["Amazonbot","Anthropic-ai","Applebot-Extended","AwarioRssBot","AwarioSmartBot","Bytespider","CCBot","ChatGPT-User","ClaudeBot","Claude-Web","Cohere-ai","DataForSeoBot","Google-Extended","GPTBot","ImagesiftBot","Magpie-crawler","Omgili","Omgilibot","Peer39_crawler","Peer39_crawler/1.0","PerplexityBot","YouBot","Googlebot","Googlebot-Image","Googlebot-News","Googlebot-Video","AdSense","AdsBot-Google","AdsBot","Google-InspectionTool","APIs-Google","Feedfetcher-Google","Google-NotebookLM","facebookexternalhit","Facebot","meta-externalagent","Facebook","Meta","*"]}
{"domain":"novinhabucetuda.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["*","Googlebot","Googlebot-Video"]}
{"domain":"ufrgs.br","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ETIMEDOUT","attempts":["ETIMEDOUT","ETIMEDOUT"]}
{"domain":"bestguiltypleasure.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"workman.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"buzzarab.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Mediapartners-Google","*"]}
{"domain":"yupmanga.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"nowtv.com.tr","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":27,"stanza_tokens":["Googlebot","googlebot-image","googlebot-mobile","MSNBot","Bingbot","DuckDuckBot","Twitterbot","Yandex","Slurp","Teoma","twiceler","Gigabot","Scrubby","Robozilla","Nutch","ia_archiver","baiduspider","naverbot","yeti","yahoo-mmcrawler","psbot","asterias","yahoo-blogs","dotbot","TweetmemeBot","bitlybot","*"]}
{"domain":"sexallarab.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"classificadosx.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":9,"stanza_tokens":["*","Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot","Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot","Gemini","SemrushBot","SemrushBot-SA","Screaming Frog SEO Spider","AhrefsBot","deepcrawl"]}
{"domain":"awsdns-20.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"chileatiende.gob.cl","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"crossref.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"avsee.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"uppcl.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"gravitational.io","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"wo-cloud.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"tamilsexzone.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"unibe.ch","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","Googlebot-Image","AhrefsBot","AhrefsSiteAudit","cortex","BLEXBot","Yandex"]}
{"domain":"banglachotikahinii.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"educacao.sp.gov.br","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["UNABLE_TO_VERIFY_LEAF_SIGNATURE",403]}
{"domain":"thebalance.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_TLS_CERT_ALTNAME_INVALID","attempts":["ERR_TLS_CERT_ALTNAME_INVALID","ERR_TLS_CERT_ALTNAME_INVALID"]}
{"domain":"zazaza.me","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"sexindrag.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"tecnomat.it","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"zellbury.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","adsbot-google"]}
{"domain":"alerts.in.ua","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Yandex"]}
{"domain":"esy1.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"suramericana.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"victronenergy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"downloadwella.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"xdate.ch","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"otter.ai","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"backend-hypertube-app.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"google.la","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"arabshentai.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"2ch2.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"maxpreps.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Exabot"]}
{"domain":"wcd.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"elmaz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["Googlebot-Image"]}
{"domain":"niconsul.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"thehackernews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"okcupid.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"speedostream1.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"magento.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"readwrite.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tamilsexvids.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":14,"stanza_tokens":["*","Nuclei","WikiDo","Riddler","PetalBot","Zoominfobot","Go-http-client","Node/simplecrawler","CazoodleBot","dotbot/1.0","Gigabot","Barkrowler","BLEXBot","magpie-crawler"]}
{"domain":"cas.cn","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"winner.bet","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"romanticgolpo.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","Google-adstxt"]}
{"domain":"shcilestamp.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"UND_ERR_SOCKET","attempts":["UND_ERR_SOCKET","UND_ERR_SOCKET"]}
{"domain":"pelotalibretv.su","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tnebnet.org","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"vorwerk-digital.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ENOTFOUND"]}
{"domain":"meteored.cl","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"google.gp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"helpguide.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":2,"stanza_tokens":["GPTBot","*"]}
{"domain":"bplay.bet.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sidrachain.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"cdn-website.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["ENOTFOUND",403]}
{"domain":"aaapay1.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["ENOTFOUND",200]}
{"domain":"riovagas.com.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","AmazonAdBot"]}
{"domain":"zd-dns.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"spicymms.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"telecomsvc.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"aagag.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["anthropic-ai"],"open":[],"path_restricted":["ClaudeBot","Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":10,"stanza_tokens":["*","Mediapartners-Google","GPTBot","Google-Extended","FacebookBot","CCBot","anthropic-ai","PerplexityBot","Bytespider","Applebot-Extended"]}
{"domain":"tuazar.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"datacamp.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"omny.fm","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"movieffm.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"uni-muenster.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xnxx-arabi.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"cbl.gov.ly","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mangolibreria.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"mobizil.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"hoathinh3d.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"suki-kira.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sapphireonline.pk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":36,"stanza_tokens":["Googlebot","Googlebot-Image","*","AdsBot-Google","MJ12bot","BLEXBot","Baiduspider","EasouSpider","Exabot","yandexbot","trovitbot","trovitBot","sitecheck.internetseer.com","Zealbot","MSIECrawler","SiteSnagger","WebStripper","WebCopier","Fetch","Offline Explorer","Teleport","TeleportPro","WebZIP","linko","HTTrack","Microsoft.URL.Control","Xenu","larbin","libwww","ZyBORG","Download Ninja","WebReaper","ZumBot","facebookexternalhit/1.1","facebookexternalhit/1.0","facebookexternalhit/*"]}
{"domain":"leaktube.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ufmg.br","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ztat.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"pgsgames.games","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"betclic.ci","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"nutanix.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":16,"stanza_tokens":["*","Googlebot","Yeti","Daumoa","EtaoSpider","MegaIndex.ru/2.0","MegaIndex.ru/","MegaIndex.com/2.0","MegaIndex.com/","+https://www.megaindex.ru/","+http://megaindex.com/crawler","+http://megaindex.com/","Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)","Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://www.megaindex.com/?tab=linkAnalyze)","Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/)","Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://www.megaindex.com/)"]}
{"domain":"azureserv.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bonappetit.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","FacebookExternalHit","Twitterbot","Pinterestbot","LinkedInBot","TikTokSpider","Storebot-Google","Google-InspectionTool","AmazonAdBot","Google-Extended","Google-CloudVertexBot","GoogleOther","Applebot-Extended","Amazonbot","meta-webindexer","meta-externalagent","meta-externalfetcher","ClaudeBot","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","cohere-training-data-crawler","cohere-ai","CCBot","PanguBot","PetalBot","Bytespider","Diffbot","DuckAssistBot","MistralAI-User","Timpibot","Webzio","Webzio-Extended","archive.org_bot","ia_archiver","ia_archiver-web.archive.org","heritrix"]}
{"domain":"interestingengineering.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]}],"stanza_token_count":33,"stanza_tokens":["*","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot-Extended","Bytespider","CCBot","Claude-Web","ClaudeBot","cohere-ai","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Google-Extended","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","Meta-ExternalAgent","Meta-ExternalFetcher","omgili","omgilibot","PetalBot","Scrapy","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot"]}
{"domain":"fcf.cat","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":7,"stanza_tokens":["GPTBot","Google-Extended","ClaudeBot","anthropic-ai","PerplexityBot","Perplexity-User","CCBot"]}
{"domain":"blockchain.info","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","proximic"]}
{"domain":"interbet.co.za","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"jadwalnonton.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["Googlebot","*"]}
{"domain":"gwdg.de","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"24-hds.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"ondigitalocean.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"veed.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"tv-tokyo.co.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"reklama5.mk","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["Googlebot","msnbot","Bingbot","SemrushBot","AhrefsBot","PetalBot","Sogou web spider","Sogou inst spider","contabo","Baiduspider"]}
{"domain":"piojm.tech","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"kdreams.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"teamstoday.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"punjabpolice.gov.pk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"facct.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gdpr-info.eu","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"kgirls.net","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"dizy.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"q4cdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","attempts":["ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE","ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE"]}
{"domain":"matchwithneighbors.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":5,"stanza_tokens":["*","Googlebot-Image","Baiduspider","ia_archiver","GPTBot"]}
{"domain":"allianz.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"google.tm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"gpb.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"betswiz.in","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"twword.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bisp.gov.pk","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"sat.co.id","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"ENOTFOUND"]}
{"domain":"rexporn.st","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["ia_archiver","*"]}
{"domain":"typingerz.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"au.dk","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"betjili.live","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"jappy.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"eronity.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"viranet.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"nankankeiba.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["ENOTFOUND",404]}
{"domain":"google.com.vc","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"kuccps.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"officestation.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"dreammail.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"feedtofap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"warau.jp","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":[200]}
{"domain":"ethiolottery.et","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"havadurumu15gunluk.xyz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"telmexla.net.co","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"8888.porn","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"up-manga.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"sangtacviet.app","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"thenkiri.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"nunuporn.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"canalturf.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":3,"stanza_tokens":["Googlebot","AdsBot-Google","Googlebot-Image"]}
{"domain":"steadfast.com.bd","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"rusporn.vip","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mobile-tracker-free.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,404]}
{"domain":"education.ne.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"music-name-servers.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"petsmart.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Googlebot","Googlebot-image","PerplexityBot","OAI-SearchBot"]}
{"domain":"pangkalantoto.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"reviewjournal.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]}],"stanza_token_count":7,"stanza_tokens":["*","GPTBot","ChatGPT-User","PerplexityBot","Perplexity-User","SimpleFeed","AmazonAdBot"]}
{"domain":"set.gov.py","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"locanto.com.mx","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":174,"stanza_tokens":["*","msnbot","bingbot","Slurp","Mediapartners-Google","RankSonicBot","spbot","A6-Indexer/1.0","sistrix","trovitBot","Lipperhey","Sosospider","ia_archiver","ia_archiver/1.6","appi","BoeserSeoSpider","StackRambler","Sphider","Jyxobot/1","Jyxobot","gigabot","ccbot","dotbot","dotbot/1.0","MJ12bot","shopwiki","NaverBot-1.0","NaverBot","grub-client","grub","WebZip","larbin","b2w/0.1","Copernic","psbot","Python-urllib","NetMechanic","URL_Spider_Pro","CherryPicker","EmailCollector","EmailSiphon","WebBandit","EmailWolf","ExtractorPro","CopyRightCheck","Crescent","SiteSnagger","ProWebWalker","CheeseBot","LNSpiderguy","Mozilla","mozilla","mozilla/3","mozilla/4","mozilla/5","Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)","Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)","Mozilla/4.0 (compatible; MSIE 4.0; Windows 98)","Mozilla/4.0 (compatible; MSIE 4.0; Windows XP)","Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000)","Alexibot","Teleport","TeleportPro","MIIxpc","Telesoft","Website Quester","moget/2.1","WebZip/4.0","WebStripper","WebSauger","WebCopier","NetAnts","Mister PiX","WebAuto","TheNomad","WWW-Collector-E","RMA","libWeb/clsHTTP","asterias","httplib","turingos","spanner","InfoNaviRobot","Harvest/1.5","Bullseye/1.0","Mozilla/4.0 (compatible; BullsEye; Windows 95)","Crescent Internet ToolPak HTTP OLE Control v.1.0","CherryPickerSE/1.0","CherryPickerElite/1.0","WebBandit/3.50","NICErsPRO","DittoSpyder","Foobot","WebmasterWorldForumBot","SpankBot","BotALot","lwp-trivial/1.34","lwp-trivial","BunnySlippers","URLy Warning"]}
{"domain":"awsdns-54.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"tlivecdn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ERR_TLS_CERT_ALTNAME_INVALID","TimeoutError"]}
{"domain":"awsdns-52.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"keiba.go.jp","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","UND_ERR_CONNECT_TIMEOUT"]}
{"domain":"nr-ops.net","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"downloadpage.xyz","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"aka-ns.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"lapalabradeldia.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"mk.co.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":["OAI-SearchBot"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":["Claude-User","Claude-SearchBot"],"path_restricted":[]}],"stanza_token_count":17,"stanza_tokens":["GPTBot","SemrushBot","AhrefsBot","Bytespider","ChatGPT","CCBot","ChatGPT-User","anthropic-ai","ClaudeBot","Omgilibot","Omgili","FacebookBot","Diffbot","ImagesiftBot","cohere-ai","CLAUDE-WEB","*"]}
{"domain":"ich-tanke.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["AmazonAdBot","Mediapartners-Google","EmailCollector","GagaRobot","vscooter","Roverbot*","mirago","psbot","MSIECrawler","WebCapture*","WebSauger*","Teleport*","Webwhacker*","Webzip*","WebCopier","Net Attache*","WebReaper*","SiteSnagger*","HTTrack*","*"]}
{"domain":"allure.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"anthropic","blocked":["ClaudeBot","Claude-User","Claude-SearchBot"],"open":[],"path_restricted":["anthropic-ai"]}],"stanza_token_count":38,"stanza_tokens":["*","FacebookExternalHit","Twitterbot","Pinterestbot","LinkedInBot","TikTokSpider","Storebot-Google","Google-InspectionTool","AmazonAdBot","Google-Extended","Google-CloudVertexBot","GoogleOther","Applebot-Extended","Amazonbot","meta-webindexer","meta-externalagent","meta-externalfetcher","ClaudeBot","Claude-SearchBot","Claude-User","PerplexityBot","Perplexity-User","cohere-training-data-crawler","cohere-ai","CCBot","PanguBot","PetalBot","Bytespider","Diffbot","DuckAssistBot","MistralAI-User","Timpibot","Webzio","Webzio-Extended","archive.org_bot","ia_archiver","ia_archiver-web.archive.org","heritrix"]}
{"domain":"service-public.gouv.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"awsdns-03.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"alpha-hen.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"redlights.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"minhchinh.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"drbokep.page","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"eprocure.gov.bd","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"vidrey.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"ocregister.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":68,"stanza_tokens":["*","AI2Bot","Ai2Bot-Dolma","Amazonbot","Amzn-SearchBot","anthropic-ai","Applebot","Applebot-Extended","Archive.org_bot","Baiduspider","Brightbot 1.0","Bytespider","carbon-umbrella-bot","CCBot","ChatGLM-Spider","ChatGPT-User","ClaudeBot","Claude-Searchbot","Claude-User","Claude-Web","Cohere-training-data-crawler","Cotoyogi","Dalvik","Datenbank Crawler","Diffbot","DocSpotBot","DuckAssistBot","FacebookBot","GnowitNewsbot","Google-Extended","GPTBot","grok-crawler","ia_archiver-web.archive.org","ICC-Crawler","imageSpider","Kangaroo Bot","laion-huggingface-processor","LCC","Mantisbot","Meta-ExternalAgent","MistralAI-User","netEstate Imprint Crawler","OAI-SearchBot","omgili","PanguBot","PerplexityBot","Perplexity-User","PetalBot","SBIntuitionsBot","SirdataBot","Spider","Thinkbot","Timpibot","VelenPublicWebCrawler","Vivaldi","Webzio-extended","YandexBot","YouBot","AmazonAdBot","facebookexternalhit","grapeshot","Mozilla/5.0 (compatible; EchoboxBot/1.0; hash/w4mwnpbXf3MFAbxOkJRw; +http://www.echobox.com)","Twitterbot","Mozilla/5.0 (compatible; NewsRoom.BI/0.1; +http://www.newsroom.bi/bot.html)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36(KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36 (compatible; mrfCompass-Booldog/1.0)","Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; mrfCompass-Marshall/1.0)","Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Jukebox/1.0)"]}
{"domain":"udiseplus.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"sarapbabe.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"itemmania.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"orekou.net","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":["TimeoutError",404]}
{"domain":"mob.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xvideosporno.blog.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"betpawa.cd","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","ClaudeBot","anthropic-ai","PerplexityBot","Amazonbot","Bytespider","cohere-ai"]}
{"domain":"dailystar.co.uk","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","OAI-SearchBot"],"open":[],"path_restricted":["ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":22,"stanza_tokens":["*","daumoa","Sosospider","rogerbot","semetrical","Googlebot-News","AhrefsBot","GPTBot","OAI-SearchBot","Applebot-Extended","PerplexityBot","Perplexity-ai","CCBot","YouBot","ClaudeBot","anthropic-ai","Claude-Web","grapeshot","bingbot","AmazonAdBot","ozone","Meta-ExternalAgent"]}
{"domain":"naijared.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"fastvideosave.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":0,"stanza_tokens":[]}
{"domain":"nishatlinen.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","adsbot-google"]}
{"domain":"cdnfonts.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"ylilauta.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":97,"stanza_tokens":["*","AddSearchBot","AI2Bot","Ai2Bot-Dolma","aiHitBot","Amazonbot","Andibot","anthropic-ai","Applebot","Applebot-Extended","archive.org_bot","Awario","bedrockbot","bigsur.ai","Brightbot 1.0","Bytespider","CCBot","ChatGPT Agent","ChatGPT-User","Claude-SearchBot","Claude-User","Claude-Web","ClaudeBot","CloudVertexBot","cohere-ai","cohere-training-data-crawler","Cotoyogi","Crawlspace","Datenbank Crawler","Devin","Diffbot","DuckAssistBot","Echobot Bot","EchoboxBot","FacebookBot","facebookexternalhit","Factset_spyderbot","FirecrawlAgent","FriendlyCrawler","Gemini-Deep-Research","Google-CloudVertexBot","Google-Extended","Google-Firebase","GoogleAgent-Mariner","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","LinerBot","meta-externalagent","Meta-ExternalAgent","meta-externalfetcher","Meta-ExternalFetcher","MistralAI-User","MistralAI-User/1.0","MyCentralAIScraperBot","netEstate Imprint Crawler","NovaAct","OAI-SearchBot","omgili","omgilibot","OpenAI","Operator","PanguBot","Panscient","panscient.com","Perplexity-User","PerplexityBot","PetalBot","PhindBot","Poseidon Research Crawler","QualifiedBot","QuillBot","quillbot.com","SBIntuitionsBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","ShapBot","Sidetrade indexer bot","Thinkbot","TikTokSpider","Timpibot","VelenPublicWebCrawler","WARDBot","Webzio-Extended","wpbot","YaK","YandexAdditional","YandexAdditionalBot","YouBot"]}
{"domain":"porno18.site","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":18,"stanza_tokens":["*","BLEXBot","AhrefsBot","Vagabondo","SEOkicks-Robot","ia_archiver","archive.org_bot","special_archiver","MJ12bot","heritrix","netEstate NE Crawler","sistrix","WBSearchBot","QuerySeekerSpider","proximic","SiteExplorer","SemrushBot","SemrushBot-SA"]}
{"domain":"soccervital.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bebesong.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"espacioapk.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"erotikchat4free.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":11,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","ia_archiver"]}
{"domain":"hentaivietsub.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"marketsandmarkets.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["GPTBot","ChatGPT-User","Google-Extended","ClaudeBot","Anthropic-User-Agent","PerplexityBot","cohere-ai","aicodebot","YouBot","*"]}
{"domain":"smartnews.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"CCBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"PARTIAL","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":55,"stanza_tokens":["*","Twitterbot","facebookexternalhit","AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","Brightbot 1.0","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","cohere-training-data-crawler","Crawlspace","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","imgproxy","ISSCyberRiskCrawler","Kangaroo Bot","Meta-ExternalAgent","Meta-ExternalFetcher","NewsNow","news-please","NovaAct","OAI-SearchBot","omgili","omgilibot","Operator","PanguBot","Perplexity-User","PerplexityBot","PetalBot","Scrapy","SemrushBot-OCOB","SemrushBot-SWA","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot"]}
{"domain":"qeseh.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"flashscore.gr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":20,"stanza_tokens":["*","SmartViper","Mediapartners-Google","CCBot","Omgilibot","Omgili","FacebookBot","Meta-ExternalAgent","Meta-ExternalFetcher","Diffbot","DuckAssistBot","AI2Bot","Bytespider","ImagesiftBot","Kangaroo Bot","cohere-ai","Timpibot","Webzio-Extended","YouBot","AmazonAdBot"]}
{"domain":"quartier-rouge.be","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"betpawa.co.zm","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","ClaudeBot","anthropic-ai","PerplexityBot","Amazonbot","Bytespider","cohere-ai"]}
{"domain":"forogore.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"blend.school","fetch_ok":true,"usable":true,"status":200,"wildcard":"FULL_BLOCK","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ChatGPT-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"anthropic-ai":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"CCBot":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"FULL_BLOCK"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ebharatgas.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"shkolo.bg","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"tamo.lt","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"sex-masry.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"xosothantai.mobi","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":14,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent","ia_archiver","SemrushBot","SemrushBot-SA","SEOquake"]}
{"domain":"imssms.org","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"fullcast.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"betfair.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"betpawa.mw","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","ClaudeBot","anthropic-ai","PerplexityBot","Amazonbot","Bytespider","cohere-ai"]}
{"domain":"darknaija.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"detran.sp.gov.br","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"absoluporn.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"ECONNREFUSED","attempts":["ECONNREFUSED","ECONNREFUSED"]}
{"domain":"arbpay.me","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"bizimhesap.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"google.dj","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":5,"stanza_tokens":["*","Yandex","AdsBot-Google","facebookexternalhit","Twitterbot"]}
{"domain":"artsy.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"locanto.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]}],"stanza_token_count":174,"stanza_tokens":["*","msnbot","bingbot","Slurp","Mediapartners-Google","RankSonicBot","spbot","A6-Indexer/1.0","sistrix","trovitBot","Lipperhey","Sosospider","ia_archiver","ia_archiver/1.6","appi","BoeserSeoSpider","StackRambler","Sphider","Jyxobot/1","Jyxobot","gigabot","ccbot","dotbot","dotbot/1.0","MJ12bot","shopwiki","NaverBot-1.0","NaverBot","grub-client","grub","WebZip","larbin","b2w/0.1","Copernic","psbot","Python-urllib","NetMechanic","URL_Spider_Pro","CherryPicker","EmailCollector","EmailSiphon","WebBandit","EmailWolf","ExtractorPro","CopyRightCheck","Crescent","SiteSnagger","ProWebWalker","CheeseBot","LNSpiderguy","Mozilla","mozilla","mozilla/3","mozilla/4","mozilla/5","Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)","Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)","Mozilla/4.0 (compatible; MSIE 4.0; Windows 98)","Mozilla/4.0 (compatible; MSIE 4.0; Windows XP)","Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000)","Alexibot","Teleport","TeleportPro","MIIxpc","Telesoft","Website Quester","moget/2.1","WebZip/4.0","WebStripper","WebSauger","WebCopier","NetAnts","Mister PiX","WebAuto","TheNomad","WWW-Collector-E","RMA","libWeb/clsHTTP","asterias","httplib","turingos","spanner","InfoNaviRobot","Harvest/1.5","Bullseye/1.0","Mozilla/4.0 (compatible; BullsEye; Windows 95)","Crescent Internet ToolPak HTTP OLE Control v.1.0","CherryPickerSE/1.0","CherryPickerElite/1.0","WebBandit/3.50","NICErsPRO","DittoSpyder","Foobot","WebmasterWorldForumBot","SpankBot","BotALot","lwp-trivial/1.34","lwp-trivial","BunnySlippers","URLy Warning"]}
{"domain":"ulluwebseries.one","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xlxx2.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"congobet.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot","anthropic-ai"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":17,"stanza_tokens":["*","anthropic-ai","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","ClaudeBot","cohere-ai","Diffbot","FacebookBot","GPTBot","ImagesiftBot","Meta-ExternalAgent","Meta-ExternalFetcher","Omgilibot","PerplexityBot","Timpibot"]}
{"domain":"assertcom.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"playwekings.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"easybell.de","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"megoplan.ru","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"kaldagan.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ntnu.no","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"research-panel.jp","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"cvut.cz","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"yuvideos.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"bpexch.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"barotem.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"airbnb.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"animeruka.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,403]}
{"domain":"ardentnews.co.kr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","bingbot"]}
{"domain":"mindbodyonline.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ogomovies1.com.pk","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"retailrocket.ru","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","PerplexityBot","Google-Extended","Applebot-Extended","CCBot"]}
{"domain":"downringtone.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bt.com.ve","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"xn--42cah7d0cxcvbbb9x.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"EXPLICIT_ALLOW","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":7,"stanza_tokens":["*","GPTBot","Google-Extended","Applebot-Extended","ClaudeBot","CCBot","Bytespider"]}
{"domain":"cisf.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"yoox.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"awsdns-50.co.uk","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"ahindian.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"pagalnew.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"lectio.dk","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"yieldlove-ad-serving.net","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"elephantbet.co.ao","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"listrakbi.com","fetch_ok":false,"usable":false,"reason":"no_robots_txt","status":404,"error":null,"attempts":[404,"TimeoutError"]}
{"domain":"79king1.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":[403,"TimeoutError"]}
{"domain":"samsungweather.com","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","TimeoutError"]}
{"domain":"upbhulekh.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"app-us1.com","fetch_ok":false,"usable":false,"reason":"other","status":530,"error":null,"attempts":["TimeoutError",530,"TimeoutError"]}
{"domain":"samsungdive.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"filmexxx.live","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nothingtech.link","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"ruso-turist.ru","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}
{"domain":"odisha.gov.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"yealink.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"aeolservice.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"betpawa.bj","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"PARTIAL","effective":"PARTIAL"},"ClaudeBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"PARTIAL","effective":"PARTIAL"},"Google-Extended":{"explicit":"PARTIAL","effective":"PARTIAL"},"PerplexityBot":{"explicit":"PARTIAL","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"PARTIAL","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"PARTIAL","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":10,"stanza_tokens":["*","GPTBot","ChatGPT-User","Google-Extended","ClaudeBot","anthropic-ai","PerplexityBot","Amazonbot","Bytespider","cohere-ai"]}
{"domain":"zhaopin.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"animedekho.app","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"reviews.io","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":11,"stanza_tokens":["Cliqzbot","MJ12bot","rogerbot","dotbot","Alexibot","SurveyBot","Xenu's","Xenu's Link Sleuth 1.1c","AspiegelBot","omgili","*"]}
{"domain":"napiszex.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"ladyxena.com","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"sexybluefilm.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"gharsoaps.shop","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":2,"stanza_tokens":["*","adsbot-google"]}
{"domain":"speed-manga.net","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":[],"path_restricted":["OAI-SearchBot","ChatGPT-User"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"1goplus.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"ruta1000.com.ar","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"bussimulator.id","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"stipepay.com","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Applebot-Extended":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Bytespider":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"meta-externalagent":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Amazonbot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"uxzw.club","fetch_ok":false,"usable":false,"reason":"no_dns","status":null,"error":"ENOTFOUND","attempts":["ENOTFOUND","ENOTFOUND"]}
{"domain":"tvguia.es","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot","ChatGPT-User"],"open":[],"path_restricted":["OAI-SearchBot"]},{"family":"anthropic","blocked":["ClaudeBot"],"open":[],"path_restricted":["Claude-User","Claude-SearchBot","anthropic-ai"]},{"family":"perplexity","blocked":["PerplexityBot"],"open":[],"path_restricted":["Perplexity-User"]}],"stanza_token_count":9,"stanza_tokens":["GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","PerplexityBot","CCBot","Amazonbot","Bytespider","*"]}
{"domain":"chronopost.fr","fetch_ok":true,"usable":true,"status":200,"wildcard":"PARTIAL","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"PARTIAL"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"ChatGPT-User":{"explicit":"ABSENT","effective":"PARTIAL"},"ClaudeBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-User":{"explicit":"ABSENT","effective":"PARTIAL"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"PARTIAL"},"anthropic-ai":{"explicit":"ABSENT","effective":"PARTIAL"},"Google-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"PerplexityBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Perplexity-User":{"explicit":"ABSENT","effective":"PARTIAL"},"CCBot":{"explicit":"ABSENT","effective":"PARTIAL"},"Applebot-Extended":{"explicit":"ABSENT","effective":"PARTIAL"},"Bytespider":{"explicit":"ABSENT","effective":"PARTIAL"},"meta-externalagent":{"explicit":"ABSENT","effective":"PARTIAL"},"Amazonbot":{"explicit":"ABSENT","effective":"PARTIAL"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"metabet.tv","fetch_ok":false,"usable":false,"reason":"bot_challenged","status":403,"error":null,"attempts":["TimeoutError",403]}
{"domain":"allsectech.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"net22.cc","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","ENOTFOUND"]}
{"domain":"theplatform.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","CERT_HAS_EXPIRED","CERT_HAS_EXPIRED"]}
{"domain":"tippmix.hu","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"allpanelexch9.co","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"looker.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"yt2mp3.sc","fetch_ok":true,"usable":true,"status":200,"wildcard":"NO_RULES","blocks_any_ai":false,"bots":{"GPTBot":{"explicit":"ABSENT","effective":"NO_RULES"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"ChatGPT-User":{"explicit":"ABSENT","effective":"NO_RULES"},"ClaudeBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-User":{"explicit":"ABSENT","effective":"NO_RULES"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"NO_RULES"},"anthropic-ai":{"explicit":"ABSENT","effective":"NO_RULES"},"Google-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"PerplexityBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Perplexity-User":{"explicit":"ABSENT","effective":"NO_RULES"},"CCBot":{"explicit":"ABSENT","effective":"NO_RULES"},"Applebot-Extended":{"explicit":"ABSENT","effective":"NO_RULES"},"Bytespider":{"explicit":"ABSENT","effective":"NO_RULES"},"meta-externalagent":{"explicit":"ABSENT","effective":"NO_RULES"},"Amazonbot":{"explicit":"ABSENT","effective":"NO_RULES"}},"gaps":[],"stanza_token_count":1,"stanza_tokens":["*"]}
{"domain":"arabx88.mobi","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ENOTFOUND"]}
{"domain":"onstove.com","fetch_ok":true,"usable":false,"reason":"html_not_robots","status":200,"error":null,"attempts":["TimeoutError","TimeoutError",200]}
{"domain":"relaysteering.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"ipv6-test.com","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","ETIMEDOUT"]}
{"domain":"adcam.org","fetch_ok":true,"usable":true,"status":200,"wildcard":"EXPLICIT_ALLOW","blocks_any_ai":true,"bots":{"GPTBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"OAI-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ChatGPT-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"ClaudeBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Claude-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Claude-SearchBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"anthropic-ai":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Google-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"PerplexityBot":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"Perplexity-User":{"explicit":"ABSENT","effective":"EXPLICIT_ALLOW"},"CCBot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Applebot-Extended":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Bytespider":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"meta-externalagent":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"},"Amazonbot":{"explicit":"FULL_BLOCK","effective":"FULL_BLOCK"}},"gaps":[{"family":"openai","blocked":["GPTBot"],"open":["OAI-SearchBot","ChatGPT-User"],"path_restricted":[]},{"family":"anthropic","blocked":["ClaudeBot"],"open":["Claude-User","Claude-SearchBot","anthropic-ai"],"path_restricted":[]}],"stanza_token_count":10,"stanza_tokens":["*","Amazonbot","Applebot-Extended","Bytespider","CCBot","ClaudeBot","CloudflareBrowserRenderingCrawler","Google-Extended","GPTBot","meta-externalagent"]}
{"domain":"masteredu.ai","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError",404]}
{"domain":"kemensos.go.id","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"nta.nic.in","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","TimeoutError","TimeoutError"]}
{"domain":"zadn.vn","fetch_ok":false,"usable":false,"reason":"fetch_error","status":null,"error":"TimeoutError","attempts":["TimeoutError","ENOTFOUND","TimeoutError"]}